[clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #221328)
via cfe-commits
cfe-commits at lists.llvm.org
Thu Oct 8 10:01:58 PDT 2026
https://github.com/asudarsa-qti updated https://github.com/llvm/llvm-project/pull/221328
>From a42d95ac8648cc09bec6be1d426b08f3de6de90a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:14:42 -0400
Subject: [PATCH 01/43] [OpenCL][CoopMat] Patch 1 - Introduce Cooperative
matrix type
---
clang/include/clang/AST/ASTContext.h | 10 +++
clang/include/clang/AST/RecursiveASTVisitor.h | 11 +++
clang/include/clang/AST/TypeBase.h | 90 +++++++++++++++++++
clang/include/clang/AST/TypeLoc.h | 34 ++++++-
clang/include/clang/AST/TypeProperties.td | 13 +++
clang/include/clang/Basic/TypeNodes.td | 1 +
.../clang/Serialization/TypeBitCodes.def | 1 +
clang/lib/AST/ASTContext.cpp | 82 +++++++++++++++++
clang/lib/AST/Type.cpp | 38 ++++++++
clang/lib/AST/TypePrinter.cpp | 15 ++++
10 files changed, 294 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index b04b633561661..539123df01298 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -264,6 +264,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
mutable llvm::ContextualFoldingSet<DependentVectorType, ASTContext &>
DependentVectorTypes;
mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
+ mutable llvm::FoldingSet<CooperativeMatrixType> CooperativeMatrixTypes;
mutable llvm::ContextualFoldingSet<DependentSizedMatrixType, ASTContext &>
DependentSizedMatrixTypes;
mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
@@ -1917,6 +1918,15 @@ class ASTContext : public RefCountedBase<ASTContext> {
QualType ElementType, unsigned NumRows, unsigned NumColumns,
std::optional<MatrixType::LayoutKind> Layout = std::nullopt) const;
+ /// Return the unique reference to the cooperative matrix type of the
+ /// specified element type and size.
+ ///
+ /// \pre \p ElementType must be a valid matrix element type (see
+ /// MatrixType::isValidElementType).
+ QualType getCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use) const;
+
/// Return the unique reference to the matrix type of the specified element
/// type and size
QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index f144c890f6ee6..791536d03f8f4 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1089,6 +1089,9 @@ DEF_TRAVERSE_TYPE(ExtVectorType, { TRY_TO(TraverseType(T->getElementType())); })
DEF_TRAVERSE_TYPE(ConstantMatrixType,
{ TRY_TO(TraverseType(T->getElementType())); })
+DEF_TRAVERSE_TYPE(CooperativeMatrixType,
+ { TRY_TO(TraverseType(T->getElementType())); })
+
DEF_TRAVERSE_TYPE(DependentSizedMatrixType, {
if (T->getRowExpr())
TRY_TO(TraverseStmt(T->getRowExpr()));
@@ -1409,6 +1412,14 @@ DEF_TRAVERSE_TYPELOC(ConstantMatrixType, {
TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
})
+DEF_TRAVERSE_TYPELOC(CooperativeMatrixType, {
+ TRY_TO(TraverseStmt(TL.getAttrScopeOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrUseOperand()));
+ TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
+})
+
DEF_TRAVERSE_TYPELOC(DependentSizedMatrixType, {
TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 6373794bb69f5..9b1c9ef646f1f 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -2708,6 +2708,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
bool isSubscriptableVectorType() const;
bool isMatrixType() const; // Matrix type.
bool isConstantMatrixType() const; // Constant matrix type.
+ bool isCooperativeMatrixType() const; // Cooperative matrix type.
bool isOverflowBehaviorType() const; // Overflow behavior type.
bool isDependentAddressSpaceType() const; // value-dependent address space qualifier
bool isObjCObjectPointerType() const; // pointer to ObjC object
@@ -4482,6 +4483,7 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
+ T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4577,6 +4579,90 @@ class ConstantMatrixType final : public MatrixType {
}
};
+/// Represents a cooperative matrix type.
+class CooperativeMatrixType final : public MatrixType {
+protected:
+ friend class ASTContext;
+
+ /// Number of rows and columns.
+ unsigned NumRows;
+ unsigned NumColumns;
+
+ /// Scope and use
+ unsigned Scope;
+ unsigned Use;
+
+ static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
+
+ CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
+ unsigned NRows, unsigned NColumns, unsigned Use,
+ QualType CanonElementType);
+
+ CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
+ unsigned Scope, unsigned NRows, unsigned NColumns,
+ unsigned Use, QualType CanonElementType);
+
+public:
+ /// Returns the number of rows in the matrix.
+ unsigned getNumRows() const { return NumRows; }
+
+ /// Returns the number of columns in the matrix.
+ unsigned getNumColumns() const { return NumColumns; }
+
+ /// Returns the scope of the matrix.
+ unsigned getScope() const { return Scope; }
+
+ /// Returns the use of the matrix.
+ unsigned getUse() const { return Use; }
+
+ /// Returns the number of elements required to embed the matrix into a vector.
+ unsigned getNumElementsFlattened() const {
+ return getNumRows() * getNumColumns();
+ }
+
+ /// Returns true if \p NumElements is a valid matrix dimension.
+ static constexpr bool isDimensionValid(size_t NumElements) {
+ return NumElements > 0 && NumElements <= MaxElementsPerDimension;
+ }
+
+ /// Return true if \p Scope is valid
+ static constexpr bool isScopeValid(size_t Scope) {
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ }
+
+ /// Return true if \p Use is valid
+ static constexpr bool isUseValid(size_t Use) {
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ }
+
+ /// Returns the maximum number of elements per dimension.
+ static constexpr unsigned getMaxElementsPerDimension() {
+ return MaxElementsPerDimension;
+ }
+
+ void Profile(llvm::FoldingSetNodeID &ID) {
+ Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ getUse(), getTypeClass());
+ }
+
+ static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
+ unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Use, TypeClass TypeClass) {
+ ID.AddPointer(ElementType.getAsOpaquePtr());
+ ID.AddInteger(NumRows);
+ ID.AddInteger(NumColumns);
+ ID.AddInteger(Scope);
+ ID.AddInteger(Use);
+ ID.AddInteger(TypeClass);
+ }
+
+ static bool classof(const Type *T) {
+ return T->getTypeClass() == CooperativeMatrix;
+ }
+};
+
/// Represents a matrix type where the type and the number of rows and columns
/// is dependent on a template.
class DependentSizedMatrixType final : public MatrixType {
@@ -8851,6 +8937,10 @@ inline bool Type::isConstantMatrixType() const {
return isa<ConstantMatrixType>(CanonicalType);
}
+inline bool Type::isCooperativeMatrixType() const {
+ return isa<CooperativeMatrixType>(CanonicalType);
+}
+
inline bool Type::isOverflowBehaviorType() const {
return isa<OverflowBehaviorType>(CanonicalType);
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index b3cb4d0f33629..24fcda8f4c7e5 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,8 +2142,10 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
+ Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
+ Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2151,26 +2153,50 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^~~~~~~~~~~~~~~~~
+ /// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2187,8 +2213,10 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
}
};
@@ -2196,6 +2224,10 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
+class CooperativeMatrixTypeLoc
+ : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType> {};
+
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 1a2a3fda0dcff..88d84d6d3234f 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -263,6 +263,19 @@ let Class = ConstantMatrixType in {
}]>;
}
+let Class = CooperativeMatrixType in {
+ def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
+ def : Property<"numColumns", UInt32> {
+ let Read = [{ node->getNumColumns() }];
+ }
+ def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+
+ def : Creator<[{
+ return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ }]>;
+}
+
let Class = DependentSizedMatrixType in {
def : Property<"rows", ExprRef> {
let Read = [{ node->getRowExpr() }];
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 700a73f669690..e6f396245070c 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,6 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<MatrixType>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 9f1da65e0f940..c9ebac63390f5 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -70,5 +70,6 @@ TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
TYPE_BIT_CODE(OverflowBehavior, OVERFLOWBEHAVIOR, 63)
+TYPE_BIT_CODE(CooperativeMatrix, COOPERATIVE_MATRIX, 64)
#undef TYPE_BIT_CODE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 58a761f11925c..6052ee26e96fb 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2215,6 +2215,17 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
break;
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(T);
+ TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
+ // The internal layout of a matrix value is implementation defined.
+ // Initially be ABI compatible with arrays with respect to alignment and
+ // size.
+ Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
+ Align = ElementInfo.Align;
+ break;
+ }
+
case Type::Builtin:
switch (cast<BuiltinType>(T)->getKind()) {
default: llvm_unreachable("Unknown builtin type!");
@@ -3546,6 +3557,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
case Type::Pipe:
case Type::BitInt:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
OS << "?";
return;
@@ -4381,6 +4393,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
case Type::ExtVector:
case Type::DependentSizedExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::ObjCObject:
@@ -4928,6 +4941,45 @@ QualType ASTContext::getConstantMatrixType(
return QualType(New, 0);
}
+QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
+ unsigned Scope, unsigned NumRows,
+ unsigned NumColumns,
+ unsigned Use) const {
+ llvm::FoldingSetNodeID ID;
+ CooperativeMatrixType::Profile(ID, ElementTy, Scope, NumRows, NumColumns, Use,
+ Type::CooperativeMatrix);
+
+ assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
+ "need a valid element type");
+ assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
+ CooperativeMatrixType::isDimensionValid(NumColumns) &&
+ "need valid matrix dimensions");
+ assert(CooperativeMatrixType::isScopeValid(Scope) &&
+ "need valid matrix scope");
+ assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
+ void *InsertPos = nullptr;
+ if (CooperativeMatrixType *MTP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ return QualType(MTP, 0);
+
+ QualType Canonical;
+ if (!ElementTy.isCanonical()) {
+ Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
+ NumRows, NumColumns, Use);
+
+ CooperativeMatrixType *NewIP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ assert(!NewIP && "Matrix type shouldn't already exist in the map");
+ (void)NewIP;
+ }
+
+ auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
+ ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
+ CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ Types.push_back(New);
+ return QualType(New, 0);
+}
+
QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
Expr *RowExpr,
Expr *ColumnExpr,
@@ -9796,6 +9848,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
return;
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
if (NotEncodedT)
*NotEncodedT = T;
return;
@@ -10764,6 +10817,18 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
+/// areCompatMatrixTypes - Return true if the two specified matrix types are
+/// compatible.
+static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
+ assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
+ return LHS->getElementType() == RHS->getElementType() &&
+ LHS->getScope() == RHS->getScope() &&
+ LHS->getNumRows() == RHS->getNumRows() &&
+ LHS->getNumColumns() == RHS->getNumColumns() &&
+ LHS->getUse() == RHS->getUse();
+}
+
bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
QualType SecondVec) {
assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
@@ -12278,6 +12343,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
RHSCan->castAs<ConstantMatrixType>()))
return LHS;
return {};
+ case Type::CooperativeMatrix:
+ if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
+ return LHS;
+ return {};
case Type::ObjCObject: {
// Check if the types are assignment compatible.
// FIXME: This should be type compatibility, e.g. whether
@@ -14642,6 +14712,17 @@ static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
return Ctx.getConstantMatrixType(getCommonElementType(Ctx, MX, MY),
MX->getNumRows(), MX->getNumColumns());
}
+ case Type::CooperativeMatrix: {
+ const auto *MX = cast<CooperativeMatrixType>(X),
+ *MY = cast<CooperativeMatrixType>(Y);
+ assert(MX->getScope() == MY->getScope());
+ assert(MX->getNumRows() == MY->getNumRows());
+ assert(MX->getNumColumns() == MY->getNumColumns());
+ assert(MX->getUse() == MY->getUse());
+ return Ctx.getCooperativeMatrixType(getCommonElementType(Ctx, MX, MY),
+ MX->getScope(), MX->getNumRows(),
+ MX->getNumColumns(), MX->getUse());
+ }
case Type::DependentSizedMatrix: {
const auto *MX = cast<DependentSizedMatrixType>(X),
*MY = cast<DependentSizedMatrixType>(Y);
@@ -14796,6 +14877,7 @@ static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
CANONICAL_TYPE(ConstantArray)
CANONICAL_TYPE(ArrayParameter)
CANONICAL_TYPE(ConstantMatrix)
+ CANONICAL_TYPE(CooperativeMatrix)
CANONICAL_TYPE(Enum)
CANONICAL_TYPE(ExtVector)
CANONICAL_TYPE(FunctionNoProto)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 4c5f2f60a9000..020564f3edfb7 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,6 +510,20 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
+CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
+ nColumns, use, canonType) {}
+
+CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : MatrixType(tc, matrixType, canonType), NumRows(nRows),
+ NumColumns(nColumns), Scope(scope), Use(use) {}
+
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -1286,6 +1300,18 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
T->getNumColumns(), T->getLayout());
}
+ QualType VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ QualType elementType = recurse(T->getElementType());
+ if (elementType.isNull())
+ return {};
+ if (elementType.getAsOpaquePtr() == T->getElementType().getAsOpaquePtr())
+ return QualType(T, 0);
+
+ return Ctx.getCooperativeMatrixType(elementType, T->getScope(),
+ T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ }
+
QualType VisitOverflowBehaviorType(const OverflowBehaviorType *T) {
QualType UnderlyingType = recurse(T->getUnderlyingType());
if (UnderlyingType.isNull())
@@ -2193,6 +2219,10 @@ class GetContainedDeducedTypeVisitor
return Visit(T->getElementType());
}
+ Type *VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+ }
+
Type *VisitFunctionProtoType(const FunctionProtoType *T) {
if (Syntactic && T->hasTrailingReturn())
return const_cast<FunctionProtoType *>(T);
@@ -3274,6 +3304,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
+ if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
+ return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
@@ -5166,6 +5198,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
return Cache::get(cast<VectorType>(T)->getElementType());
case Type::ConstantMatrix:
return Cache::get(cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return Cache::get(cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return Cache::get(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5268,6 +5302,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
case Type::ConstantMatrix:
return computeTypeLinkageInfo(
cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return computeTypeLinkageInfo(
+ cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return computeTypeLinkageInfo(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5472,6 +5509,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::FunctionProto:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 13929f6fe0800..34380e3c8965f 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -280,6 +280,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -917,6 +918,20 @@ void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
printAfter(T->getElementType(), OS);
}
+void TypePrinter::printCooperativeMatrixBefore(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printBefore(T->getElementType(), OS);
+ OS << " __attribute__((coop_mat(";
+ OS << T->getScope() << ", " << T->getNumRows() << ", ";
+ OS << T->getNumColumns() << ", " << T->getUse();
+ OS << ")))";
+}
+
+void TypePrinter::printCooperativeMatrixAfter(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printAfter(T->getElementType(), OS);
+}
+
void TypePrinter::printDependentSizedMatrixBefore(
const DependentSizedMatrixType *T, raw_ostream &OS) {
if (Policy.UseHLSLTypes) {
>From 62cba28b48b6b646a74bdb1a90bfb597093ab34c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:15:43 -0400
Subject: [PATCH 02/43] Patch 2 - Define extension macro
---
.../include/clang/Basic/OpenCLExtensions.def | 1 +
clang/lib/Headers/opencl-c-base.h | 23 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index f8af9b843f95f..89e96dca85a59 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,6 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
+OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 251894ac5a263..7b3d239299c6e 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -771,4 +771,27 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From aa1aacd18e2d3fddd6420897b49ca2a1d0958cc4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:16:39 -0400
Subject: [PATCH 03/43] Patch 3 - Add support for serialization, round-trip and
mangling
---
clang/lib/AST/ASTImporter.cpp | 11 +++++++++
clang/lib/AST/ASTStructuralEquivalence.cpp | 15 +++++++++++++
clang/lib/AST/ExprConstant.cpp | 1 +
clang/lib/AST/ItaniumMangle.cpp | 26 ++++++++++++++++++++++
clang/lib/AST/MicrosoftMangle.cpp | 21 +++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 10 +++++++++
clang/lib/Serialization/ASTWriter.cpp | 12 ++++++++++
7 files changed, 96 insertions(+)
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 4c296b6829b1b..1bf29a062287b 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2116,6 +2116,17 @@ ExpectedType clang::ASTNodeImporter::VisitConstantMatrixType(
*ToElementTypeOrErr, T->getNumRows(), T->getNumColumns(), T->getLayout());
}
+ExpectedType clang::ASTNodeImporter::VisitCooperativeMatrixType(
+ const clang::CooperativeMatrixType *T) {
+ ExpectedType ToElementTypeOrErr = import(T->getElementType());
+ if (!ToElementTypeOrErr)
+ return ToElementTypeOrErr.takeError();
+
+ return Importer.getToContext().getCooperativeMatrixType(
+ *ToElementTypeOrErr, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+}
+
ExpectedType clang::ASTNodeImporter::VisitDependentAddressSpaceType(
const clang::DependentAddressSpaceType *T) {
Error Err = Error::success();
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 029e33ae0db76..8ec1100f9f4d8 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1151,6 +1151,21 @@ bool ASTStructuralEquivalence::isEquivalent(
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
+ const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
+ // The element types must be structurally equivalent and the number of rows
+ // and columns must match.
+ if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
+ Mat2->getElementType()) ||
+ Mat1->getScope() != Mat2->getScope() ||
+ Mat1->getNumRows() != Mat2->getNumRows() ||
+ Mat1->getNumColumns() != Mat2->getNumColumns() ||
+ Mat1->getUse() != Mat2->getUse())
+ return false;
+ break;
+ }
+
case Type::FunctionProto: {
const auto *Proto1 = cast<FunctionProtoType>(T1);
const auto *Proto2 = cast<FunctionProtoType>(T2);
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index d5b8d0ed889d1..dfd84f62ada25 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -16583,6 +16583,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T,
case Type::BlockPointer:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::ObjCObject:
case Type::ObjCInterface:
case Type::ObjCObjectPointer:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 1a6f012199745..8264a11c5f35c 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2556,6 +2556,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -4532,6 +4533,31 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
Out << "E";
}
+void CXXNameMangler::mangleType(const CooperativeMatrixType *T) {
+ // Mangle cooperative matrix types as a vendor extended type:
+ // u<Len>coop_matI<Scope><Rows><Columns><Use><element type>E
+
+ mangleVendorType("coop_mat");
+
+ Out << "I";
+ auto &ASTCtx = getASTContext();
+ unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
+ llvm::APSInt Scope(BitWidth);
+ Scope = T->getScope();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Scope);
+ llvm::APSInt Rows(BitWidth);
+ Rows = T->getNumRows();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
+ llvm::APSInt Columns(BitWidth);
+ Columns = T->getNumColumns();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
+ llvm::APSInt Use(BitWidth);
+ Use = T->getUse();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Use);
+ mangleType(T->getElementType());
+ Out << "E";
+}
+
void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
// Mangle matrix types as a vendor extended type:
// u<Len>matrix_typeI<row expr><column expr><element type>E
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 74edee531e48a..612979b002b59 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3810,6 +3810,27 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T,
mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
}
+void MicrosoftCXXNameMangler::mangleType(const CooperativeMatrixType *T,
+ Qualifiers quals, SourceRange Range) {
+ QualType EltTy = T->getElementType();
+
+ llvm::SmallString<64> TemplateMangling;
+ llvm::raw_svector_ostream Stream(TemplateMangling);
+ MicrosoftCXXNameMangler Extra(Context, Stream);
+
+ Stream << "?$";
+
+ Extra.mangleSourceName("__coop_mat");
+ Extra.mangleType(EltTy, Range, QMM_Escape);
+
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getScope()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumRows()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumColumns()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getUse()));
+
+ mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
+}
+
void MicrosoftCXXNameMangler::mangleType(const DependentSizedMatrixType *T,
Qualifiers quals, SourceRange Range) {
Error(Range.getBegin(), "dependent-sized matrix type") << Range;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 36b86afeeb32f..ee39d6b883cff 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7672,6 +7672,16 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ TL.setAttrNameLoc(readSourceLocation());
+ TL.setAttrOperandParensRange(readSourceRange());
+ TL.setAttrScopeOperand(Reader.readExpr());
+ TL.setAttrRowOperand(Reader.readExpr());
+ TL.setAttrColumnOperand(Reader.readExpr());
+ TL.setAttrUseOperand(Reader.readExpr());
+}
+
void TypeLocReader::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 640047c2f9fb3..835ee0ac49071 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,6 +457,18 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ addSourceLocation(TL.getAttrNameLoc());
+ SourceRange range = TL.getAttrOperandParensRange();
+ addSourceLocation(range.getBegin());
+ addSourceLocation(range.getEnd());
+ Record.AddStmt(TL.getAttrScopeOperand());
+ Record.AddStmt(TL.getAttrRowOperand());
+ Record.AddStmt(TL.getAttrColumnOperand());
+ Record.AddStmt(TL.getAttrUseOperand());
+}
+
void TypeLocWriter::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
>From df95f8c5aaed2836aae8166a7dd6261c79891918 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:21 -0400
Subject: [PATCH 04/43] Patch 4 - Add support for semantic analysis and
checking
---
clang/include/clang/Basic/Attr.td | 8 +
.../clang/Basic/DiagnosticSemaKinds.td | 54 +++
clang/include/clang/Sema/Sema.h | 34 +-
clang/lib/Sema/SemaChecking.cpp | 310 +++++++++++++++++-
clang/lib/Sema/SemaDecl.cpp | 32 ++
clang/lib/Sema/SemaExpr.cpp | 131 +++++++-
clang/lib/Sema/SemaLookup.cpp | 1 +
clang/lib/Sema/SemaTemplate.cpp | 5 +
clang/lib/Sema/SemaTemplateDeduction.cpp | 31 ++
clang/lib/Sema/SemaType.cpp | 78 ++++-
clang/lib/Sema/TreeTransform.h | 42 +++
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 4ef0bd2b5ca7c..f66fc29fdac45 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3981,6 +3981,14 @@ def MatrixType : TypeAttr {
let PragmaAttributeSupport = 0;
}
+def CoopMatrixType : TypeAttr {
+ let Spellings = [Clang<"coop_mat">];
+ let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
+ let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
+ UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
+ let Documentation = [Undocumented];
+}
+
def Visibility : InheritableAttr {
let Clone = 0;
let Spellings = [GCC<"visibility">];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 66155d9780e64..300fda4bed280 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11988,6 +11988,60 @@ def ext_opencl_ext_vector_type_rgba_selector: ExtWarn<
def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
+// OpenCL C Cooperative Matrix.
+def err_invalid_coopmat_attr
+ : Error<"invalid argument of cooperative matrix attribute">;
+
+def err_coop_mat_not_primitive
+ : Error<"cooperative matrix attribute only supported for integer and "
+ "floating-point types">;
+
+def err_coop_mat_rol_col_size
+ : Error<"cooperative matrix row and column must have power-of-two size">;
+
+def err_coop_mem_layout_enum
+ : Error<"memory layout for cooperative matrix should be "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+
+def err_coop_element_and_pointer_type
+ : Error<"inconsistent between cooperative matrix element type and buffer "
+ "pointer type">;
+
+def err_coop_matrix_arg
+ : Error<"argument must be a valid cooperative matrix type">;
+
+def err_coop_matrix_assignment
+ : Error<"builtin return value should be assigned to cooperative matrix "
+ "type variable">;
+
+def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_A">;
+
+def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_B">;
+
+def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+
+def err_coop_matrix_element_type
+ : Error<"inconsistent cooperative matrix element type">;
+
+def err_coop_matrix_row_or_col_mismatch
+ : Error<"mismatch of cooperative matrix row or column">;
+
+def err_coop_matrix_use_type
+ : Error<"inconsistent cooperative matrix use type">;
+
+def err_unsupported_coopmat_binary_operator
+ : Error<"unsupported cooperative matrix binary operator">;
+
+def err_unsupported_coopmat_scalar_operator
+ : Error<"unsupported cooperative matrix scalar operator">;
+
+def err_ref_coopmat_var
+ : Error<"invalid reference of cooperative matrix variable">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 7fbcc4aae7d6c..0847e05509cb4 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3112,6 +3112,34 @@ class Sema final : public SemaBase {
BuiltinCountedByRefKind K);
bool BuiltinCountedByRef(CallExpr *TheCall);
+ // Coop matrix handling.
+ void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc, CallExpr *call);
+ void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc);
+ bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
+ bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult);
+ void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc);
+ ExprResult BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatScalarOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ ExprResult BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
@@ -15370,8 +15398,10 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
+
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc);
+ SourceLocation AttrLoc, Expr *Scope = nullptr,
+ Expr *Use = nullptr, bool IsCoopMat = false);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
@@ -15394,6 +15424,8 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
+ bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+
/// Build a function type.
///
/// This routine checks the function type according to C++ rules and
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0531dfa877fbd..6a74740dec507 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4154,6 +4154,27 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
break;
}
+ case Builtin::BIcoop_mat_load:
+ return BuiltinCoopMatrixLoad(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_store:
+ return BuiltinCoopMatrixStore(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_mulAdd:
+ return BuiltinCoopMatrixMulAdd(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ return BuiltinCoopMatrixBinaryOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_mul:
+ return BuiltinCoopMatrixScalarOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_neg:
+ return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17421,6 +17442,270 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
return false;
}
+// Check coop_mat_load/store buffer pointer.
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
+ unsigned PtrArgIdx) {
+ bool ArgError = false;
+ Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
+ ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+ if (PtrConv.isInvalid())
+ return true;
+ PtrExpr = PtrConv.get();
+ TheCall->setArg(PtrArgIdx, PtrExpr);
+
+ auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+ QualType ElementTy;
+ if (!PtrTy) {
+ ArgError = true;
+ } else {
+ ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
+ if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ ArgError = true;
+ }
+ }
+
+ if (ArgError) {
+ Diag(PtrExpr->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+ << PtrArgIdx + 1 << 0 << /* pointer to element ty */ 5 << /* no fp */ 0
+ << PtrExpr->getType();
+ }
+
+ return ArgError;
+}
+
+// Check coop_mat_load/store matrix element has same type with buffer pointer.
+void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
+ auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
+ if (!MTy) {
+ Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ return;
+ }
+
+ assert(isa<PointerType>(BufferType));
+ auto *PTy = BufferType->castAs<PointerType>();
+
+ if (MTy->getElementType().getUnqualifiedType() !=
+ PTy->getPointeeType().getUnqualifiedType())
+ Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+}
+
+void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
+
+ FunctionDecl *F = call->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (Fname->isStr("coop_mat_load"))
+ CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
+ MatrixLoc);
+}
+
+// Check coop_mat_load/store layout argument
+bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
+ bool ArgError = false;
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LayoutExpr);
+ if (DR) {
+ const auto *ECDHS = dyn_cast<EnumConstantDecl>(DR->getDecl());
+ if (ECDHS) {
+ if (ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+ ArgError = true;
+ } else
+ ArgError = true;
+ } else
+ ArgError = true;
+
+ if (ArgError)
+ Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+
+ return ArgError;
+}
+
+ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
+ return ExprError();
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
+ return ExprError();
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 4))
+ return ExprError();
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ return ExprError();
+ CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
+ return ExprError();
+ return CallResult;
+}
+
+void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+ FunctionDecl *F = TheCall->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (!Fname->isStr("coop_mat_mulAdd"))
+ return;
+
+ auto MC = TheCall->getArg(2);
+ auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = MC->getType()->getAs<CooperativeMatrixType>();
+ auto Loc = TheCall->getBeginLoc();
+
+ if (!MOutTy)
+ Diag(Loc, diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!MOutTy || !M2Ty)
+ return;
+
+ if (MOutTy->getUse() != 2)
+ Diag(Loc, diag::err_coop_matrix_useACC);
+
+ if (MOutTy->getElementType().getUnqualifiedType() !=
+ M2Ty->getElementType().getUnqualifiedType())
+ Diag(Loc, diag::err_coop_matrix_element_type);
+
+ if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+}
+
+bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc) {
+ auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
+ auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
+ if (!M0Ty)
+ Diag(ALoc, diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(BLoc, diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty)
+ return true;
+
+ if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+
+ if (M0Ty->getUse() != M1Ty->getUse()) {
+ Diag(ALoc, diag::err_coop_matrix_use_type);
+ return true;
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType()) {
+ Diag(ALoc, diag::err_coop_matrix_element_type);
+ return true;
+ }
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+
+ CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
+ Arg1->getBeginLoc());
+
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
+ if (ATy->isIntegerType() && CTy->isIntegerType())
+ return true;
+ if (ATy->isFloatingType() && CTy->isFloatingType())
+ return true;
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ Expr *Arg2 = TheCall->getArg(2);
+
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto *M1Ty = Arg1->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = Arg2->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+ auto Loc1 = Arg1->getBeginLoc();
+
+ if (!M0Ty)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty || !M2Ty)
+ return ExprError();
+
+ if (M0Ty->getUse() != 0)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ if (M1Ty->getUse() != 1)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ if (M2Ty->getUse() != 2)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+
+ if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
+ M2Ty->getElementType()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+
+ if (M0Ty->getNumRows() != M2Ty->getNumRows())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
+ (M0Ty->getNumColumns() != M1Ty->getNumRows()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
@@ -17431,8 +17716,9 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
- if (!MType) {
+ auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
+ auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
+ if (!ConstMType && !CoopMType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17441,11 +17727,23 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- QualType ResultType = Context.getConstantMatrixType(
- MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
+ if (ConstMType) {
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
+ if (CoopMType) {
+ QualType ResultType = Context.getCooperativeMatrixType(
+ CoopMType->getElementType(),CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
+
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index fdbe152116512..343d260fb1a26 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14102,6 +14102,25 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
+// Return true if RHSExpr is a cooperative matrix builtin call.
+bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ if (!call)
+ return false;
+ FunctionDecl *F = call->getDirectCallee();
+ if (!F)
+ return false;
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ if (!Fname)
+ return false;
+ if (Fname->getName().starts_with("coop_mat") &&
+ !Fname->getName().starts_with("coop_mat_length"))
+ return true;
+
+ return false;
+}
+
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
llvm::scope_exit ResetDeclForInitializer([this]() {
if (!this->ExprEvalContexts.empty())
@@ -14620,6 +14639,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
+
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (!VDecl->getType()->isMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
}
void Sema::ActOnInitializerError(Decl *D) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 9b3384cdbdac8..78bbb3768e725 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4675,6 +4675,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::TemplateSpecialization:
@@ -8059,6 +8060,16 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
+ if (srcTy->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns();
+ }
+
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -11398,12 +11409,14 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
if (!IsDiv &&
- (LHSTy->isConstantMatrixType() || RHSTy->isConstantMatrixType()))
+ (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+ if (IsDiv && LHSTy->isCooperativeMatrixType() && RHSTy->isArithmeticType())
+ return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
@@ -11803,6 +11816,15 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -11949,6 +11971,15 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -14008,6 +14039,32 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ }
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -15738,6 +15795,47 @@ static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
return HasVectorOfHalfType(E0) && (!E1 || HasVectorOfHalfType(E1));
}
+ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Add:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
+ Args);
+ case BO_Sub:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
+ Args);
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
+ Args);
+ case BO_Div:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_binary_operator));
+}
+
+ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_scalar_operator));
+}
+
ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
BinaryOperatorKind Opc, Expr *LHSExpr,
Expr *RHSExpr, bool ForFoldExpression) {
@@ -15774,6 +15872,17 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
if (getLangOpts().OpenCL) {
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
+ // Cooperative matrix support
+ if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ // Check matrix types for assignment.
+ if (BO_Assign == Opc) {
+ if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
+ RHSExpr->getBeginLoc()))
+ return ExprError();
+ } else
+ return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
@@ -15801,6 +15910,18 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (!LHSExpr->getType()->isMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
+ return ExprError();
+ }
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ assert(call);
+ call->setType(LHSExpr->getType());
+ CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ call);
+ CheckCoopMatrixMatMulOutput(call);
+ }
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
LHS.get()->getObjectKind() != OK_ObjCProperty) {
@@ -16460,6 +16581,10 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
+ if (Opc == UO_Minus && Ty->isMatrixType()) {
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
+ {InputExpr});
+ }
// The only legal unary operation for atomics is '&'.
if ((Opc != UO_AddrOf && Ty->isAtomicType()) ||
// OpenCL special types - image, sampler, pipe, and blocks are to be used
@@ -16538,7 +16663,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
Opc == UO_Plus && resultType->isPointerType())
break;
-
+ else if (getLangOpts().OpenCL && Opc == UO_Minus &&
+ resultType->isCooperativeMatrixType())
+ break;
return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
<< resultType << Input.get()->getSourceRange());
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index bc8a30d4e9dad..f78c65d1deb0e 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3265,6 +3265,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BitInt:
break;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 8e7433bc71281..09d27b0445554 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6393,6 +6393,11 @@ bool UnnamedLocalNoLinkageFinder::VisitConstantMatrixType(
return Visit(T->getElementType());
}
+bool UnnamedLocalNoLinkageFinder::VisitCooperativeMatrixType(
+ const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+}
+
bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
const FunctionProtoType* T) {
for (const auto &A : T->param_types()) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 2257964a4310b..f44bdf45d1c91 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2447,6 +2447,30 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
&DependentSizedMatrixType::getColumnExpr);
}
+ // (clang extension)
+ //
+ // T __attribute__((coop_mat_type(<integral constant>,
+ // <integral constant>,
+ // <integral constant>,
+ // <integral constant>)))
+ case Type::CooperativeMatrix: {
+ const auto *MP = P->castAs<CooperativeMatrixType>(),
+ *MA = A->getAs<CooperativeMatrixType>();
+ if (!MA)
+ return TemplateDeductionResult::NonDeducedMismatch;
+
+ // Check that the dimensions are the same
+ if (MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns()) {
+ return TemplateDeductionResult::NonDeducedMismatch;
+ }
+ // Perform deduction on element types.
+ return DeduceTemplateArgumentsByTypeMatch(
+ S, TemplateParams, MP->getElementType(), MA->getElementType(), Info,
+ Deduced, TDF, degradeCallPartialOrderingKind(POK),
+ /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+ }
+
// (clang extension)
//
// T __attribute__(((address_space(N))))
@@ -7087,6 +7111,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *MatType = cast<CooperativeMatrixType>(T);
+ MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
+ Depth, Used);
+ break;
+ }
+
case Type::DependentSizedMatrix: {
const DependentSizedMatrixType *MatType = cast<DependentSizedMatrixType>(T);
MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 202bd486321f9..4dcf2c6902501 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2527,9 +2527,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc) {
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc, Expr *ScopeExpr,
+ Expr *UseExpr, bool IsCoopMat) {
+ if (!IsCoopMat)
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2610,6 +2612,26 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
+ if (IsCoopMat) {
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr)
+ << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
@@ -5973,6 +5995,23 @@ static void fillAtomicQualLoc(AtomicTypeLoc ATL, const DeclaratorChunk &Chunk) {
ATL.setParensRange(SourceRange());
}
+static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
+ const ParsedAttributesView &Attrs) {
+ for (const ParsedAttr &AL : Attrs) {
+ if (AL.getKind() == ParsedAttr::AT_CoopMatrixType) {
+ MTL.setAttrNameLoc(AL.getLoc());
+ MTL.setAttrScopeOperand(AL.getArgAsExpr(0));
+ MTL.setAttrRowOperand(AL.getArgAsExpr(1));
+ MTL.setAttrColumnOperand(AL.getArgAsExpr(2));
+ MTL.setAttrUseOperand(AL.getArgAsExpr(3));
+ MTL.setAttrOperandParensRange(SourceRange());
+ return;
+ }
+ }
+
+ llvm_unreachable("no matrix_type attribute found at the expected location!");
+}
+
namespace {
class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
Sema &SemaRef;
@@ -6366,6 +6405,9 @@ namespace {
void VisitMatrixTypeLoc(MatrixTypeLoc TL) {
fillMatrixTypeLoc(TL, Chunk.getAttrs());
}
+ void VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
+ fillCooperativeMatrixTypeLoc(TL, Chunk.getAttrs());
+ }
void VisitTypeLoc(TypeLoc TL) {
llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -8981,23 +9023,28 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S) {
- if (!S.getLangOpts().MatrixTypes) {
+ Sema &S, bool IsCoopMat = false) {
+ if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- if (Attr.getNumArgs() != 2) {
+ unsigned int NumAttrs = IsCoopMat ? 4 : 2;
+ if (Attr.getNumArgs() != NumAttrs) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << 2;
+ << Attr << NumAttrs;
return;
}
-
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ if (IsCoopMat) {
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
+ Scope, Use, /* IsCoopMat */ true);
+ if (!T.isNull())
+ CurType = T;
+ }
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9281,6 +9328,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
attr.setUsedAsTypeAttr();
break;
+ case ParsedAttr::AT_CoopMatrixType:
+ HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ attr.setUsedAsTypeAttr();
+ break;
+
case ParsedAttr::AT_WebAssemblyFuncref: {
if (!HandleWebAssemblyFuncrefAttr(state, type, attr))
attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 73367d6bea063..498551937b85d 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1048,6 +1048,12 @@ class TreeTransform {
unsigned NumColumns,
SourceLocation AttributeLoc);
+ /// Build a new cooperative matrix type given the element type and
+ /// scope and use and dimensions.
+ QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use);
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
@@ -6312,6 +6318,34 @@ TreeTransform<Derived>::TransformConstantMatrixType(TypeLocBuilder &TLB,
return Result;
}
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformCooperativeMatrixType(
+ TypeLocBuilder &TLB, CooperativeMatrixTypeLoc TL) {
+ const CooperativeMatrixType *T = TL.getTypePtr();
+ QualType ElementType = getDerived().TransformType(T->getElementType());
+ if (ElementType.isNull())
+ return QualType();
+
+ QualType Result = TL.getType();
+ if (getDerived().AlwaysRebuild() || ElementType != T->getElementType()) {
+ Result = getDerived().RebuildCooperativeMatrixType(
+ ElementType, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ if (Result.isNull())
+ return QualType();
+ }
+
+ CooperativeMatrixTypeLoc NewTL = TLB.push<CooperativeMatrixTypeLoc>(Result);
+ NewTL.setAttrNameLoc(TL.getAttrNameLoc());
+ NewTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+ NewTL.setAttrScopeOperand(TL.getAttrScopeOperand());
+ NewTL.setAttrRowOperand(TL.getAttrRowOperand());
+ NewTL.setAttrColumnOperand(TL.getAttrColumnOperand());
+ NewTL.setAttrUseOperand(TL.getAttrUseOperand());
+
+ return Result;
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::TransformDependentSizedMatrixType(
TypeLocBuilder &TLB, DependentSizedMatrixTypeLoc TL) {
@@ -18240,6 +18274,14 @@ QualType TreeTransform<Derived>::RebuildConstantMatrixType(
AttributeLoc);
}
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildCooperativeMatrixType(
+ QualType ElementType, unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use) {
+ return SemaRef.Context.getCooperativeMatrixType(ElementType, Scope, NumRows,
+ NumColumns, Use);
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::RebuildDependentSizedMatrixType(
QualType ElementType, Expr *RowExpr, Expr *ColumnExpr,
>From c490cfba7c0c02a4c29550143a41b355b56cc5d2 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:56 -0400
Subject: [PATCH 05/43] Patch 5 - Emit LLVM IR and intrinsics
---
clang/include/clang/Basic/Builtins.td | 67 +++++++
clang/lib/CodeGen/CGBuiltin.cpp | 277 ++++++++++++++++++++++++++
clang/lib/CodeGen/CGDebugInfo.cpp | 29 +++
clang/lib/CodeGen/CGDebugInfo.h | 1 +
clang/lib/CodeGen/CGExpr.cpp | 61 ++++++
clang/lib/CodeGen/CGExprScalar.cpp | 14 ++
clang/lib/CodeGen/CodeGenFunction.cpp | 2 +
clang/lib/CodeGen/CodeGenFunction.h | 9 +
clang/lib/CodeGen/CodeGenTBAA.cpp | 2 +-
clang/lib/CodeGen/CodeGenTypes.cpp | 33 ++-
clang/lib/CodeGen/ItaniumCXXABI.cpp | 2 +
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +
clang/lib/CodeGen/QualTypeMapper.h | 1 +
13 files changed, 502 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 7a59aaa133d52..ab73570c9bbcb 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,6 +1931,73 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
+// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+def CoopMatLoad : Builtin {
+ let Spellings = ["coop_mat_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatStore : Builtin {
+ let Spellings = ["coop_mat_store"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatMulAdd : Builtin {
+ let Spellings = ["coop_mat_mulAdd"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryAdd : Builtin {
+ let Spellings = ["coop_mat_binary_add"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinarySub : Builtin {
+ let Spellings = ["coop_mat_binary_sub"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryMul : Builtin {
+ let Spellings = ["coop_mat_binary_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryDiv : Builtin {
+ let Spellings = ["coop_mat_binary_div"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarMul : Builtin {
+ let Spellings = ["coop_mat_scalar_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarNeg : Builtin {
+ let Spellings = ["coop_mat_scalar_neg"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatInit : Builtin {
+ let Spellings = ["coop_mat_init"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatLength : Builtin {
+ let Spellings = ["coop_mat_length"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(...)";
+}
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 233609605fb21..6156212d20a66 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2759,6 +2759,54 @@ static void ClearPadding(CodeGenFunction &CGF, Address Src,
}
}
+static std::string getSPIRVBuiltinName(unsigned BuiltinID,
+ bool IsFloat = false) {
+ switch (BuiltinID) {
+ case Builtin::BIcoop_mat_load:
+ return "__spirv_CooperativeMatrixLoadKHR";
+ case Builtin::BIcoop_mat_store:
+ return "__spirv_CooperativeMatrixStoreKHR";
+ case Builtin::BIcoop_mat_mulAdd:
+ return "__spirv_CooperativeMatrixMulAddKHR";
+ case Builtin::BIcoop_mat_binary_add:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFAdd" : "__spirv_IAdd";
+ case Builtin::BIcoop_mat_binary_sub:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFSub" : "__spirv_ISub";
+ case Builtin::BIcoop_mat_binary_mul:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFMul" : "__spirv_IMul";
+ case Builtin::BIcoop_mat_binary_div:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFDiv" : "__spirv_IDiv";
+ case Builtin::BIcoop_mat_scalar_mul:
+ return "__spirv_CooperativeMatrixScalarMulKHR";
+ case Builtin::BIcoop_mat_scalar_neg:
+ return "__spirv_CooperativeMatrixScalarNeg";
+ case Builtin::BIcoop_mat_init:
+ return "__spirv_CompositeConstruct";
+ case Builtin::BIcoop_mat_length:
+ return "__spirv_CooperativeMatrixLengthKHR";
+ }
+ assert(0 && "Unexpected Builtin");
+ return "";
+}
+
+static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
+ CodeGenModule &CGM,
+ const CooperativeMatrixType *MTy) {
+ llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
+ MTy->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
+} // namespace
+
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
@@ -4620,6 +4668,235 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Result);
}
+ case Builtin::BIcoop_mat_load: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_load without coop_mat output operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Layout = EmitScalarExpr(E->getArg(1));
+ auto Stride = EmitScalarExpr(E->getArg(2));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_store: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address Dst = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_store without coop_mat input operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Arg0 = EmitScalarExpr(E->getArg(1));
+ auto Layout = EmitScalarExpr(E->getArg(2));
+ auto Stride = EmitScalarExpr(E->getArg(3));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_mulAdd: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_mulAdd without coop_mat output operand");
+
+ auto Arg0 = E->getArg(0);
+ auto Arg1 = E->getArg(1);
+ auto Arg2 = E->getArg(2);
+ auto MA = EmitScalarExpr(Arg0);
+ auto MB = EmitScalarExpr(Arg1);
+ auto MC = EmitScalarExpr(Arg2);
+ const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
+ const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MATy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *CType = getTargetExtType(*this, CGM, MCTy);
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, CType, Int1Ty}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ llvm::Type *ElTy = ConvertType(MTy->getElementType());
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name = (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_mul: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = Arg1->getType();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_neg: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_init: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ auto Init = EmitScalarExpr(E->getArg(0));
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Init->getType()}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(InitFn, {Init});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_length: {
+ const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ ConvertType(E->getType()), {AType}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load: {
llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 93428777b9b24..408fd1ec25a1e 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3937,6 +3937,33 @@ llvm::DIType *CGDebugInfo::CreateType(const ConstantMatrixType *Ty,
return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
}
+llvm::DIType *CGDebugInfo::CreateType(const CooperativeMatrixType *Ty,
+ llvm::DIFile *Unit) {
+ // FIXME: Create another debug type for matrices
+ // For the time being, it treats it like a nested ArrayType.
+
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint32_t Align = getTypeAlignIfRequired(Ty, CGM.getContext());
+
+ // Create ranges for both dimensions.
+ llvm::SmallVector<llvm::Metadata *, 2> Subscripts;
+ auto *ColumnCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumColumns()));
+ auto *RowCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumRows()));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ ColumnCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ RowCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
+}
+
llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint32_t Align;
@@ -4396,6 +4423,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
return CreateType(cast<VectorType>(Ty), Unit);
case Type::ConstantMatrix:
return CreateType(cast<ConstantMatrixType>(Ty), Unit);
+ case Type::CooperativeMatrix:
+ return CreateType(cast<CooperativeMatrixType>(Ty), Unit);
case Type::ObjCObjectPointer:
return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
case Type::ObjCObject:
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 37520394812b4..f617623af3af4 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -244,6 +244,7 @@ class CGDebugInfo {
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ConstantMatrixType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const CooperativeMatrixType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 8323a170a1dbb..31251fdc3535c 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7694,3 +7694,64 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
}
+
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
+ llvm::Value *ScalarVal,
+ QualType CoopMatQTy)
+{
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(
+ VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(
+ VecTy->getElementCount(),
+ ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
+ BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy)
+{
+ auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
+
+ switch (Opcode) {
+ case BO_Add:
+ return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
+ : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+
+ case BO_Sub:
+ return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
+ : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+
+ case BO_Mul:
+ // mat * mat → element-wise FMul/IMul
+ // mat * scalar → handled separately (see scalar path below)
+ if (!RHS->getType()->isVectorTy()) {
+ // mat * scalar: broadcast scalar then element-wise mul
+ llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
+ return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
+ : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ }
+ return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
+ : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+
+ case BO_Div:
+ if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+
+ default:
+ llvm_unreachable("Unsupported cooperative matrix binary op");
+ }
+}
\ No newline at end of file
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 7ecaa149f7b34..4802bdff0f805 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -956,6 +956,12 @@ class ScalarExprEmitter
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
+ if (E->getType()->isCooperativeMatrixType()) { \
+ return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
+ CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), \
+ E->getType()); \
+ } \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
if (result && !promotionTy.isNull()) \
@@ -3684,6 +3690,14 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
+ QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ if (CompTy->isFloatingType())
+ return Builder.CreateFNeg(Val, "coopmat.fneg");
+ else
+ return Builder.CreateNeg(Val, "coopmat.ineg");
+ }
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 72a5808cb6229..9bdba60bd8563 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -262,6 +262,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -2584,6 +2585,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 490267aaefd86..7b69f7ff45010 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5191,6 +5191,15 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
+ /// Helper function for EmitCoopMatBinaryOp
+ llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index d33188db9da15..c0ba358207641 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,7 +332,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert(isa<ConstantMatrixType>(Ty) &&
+ assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index daed8559cc62f..6ac24c6554922 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -125,7 +125,21 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
-
+ if (T->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent a Coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+ return CoopMatType;
+ }
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -704,6 +718,23 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
MT->getNumRows() * MT->getNumColumns());
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+
+ ResultType = CoopMatType;
+ break;
+ }
case Type::FunctionNoProto:
case Type::FunctionProto:
ResultType = ConvertFunctionTypeInternal(T);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 8e9aaa1313528..75f1b595398df 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3894,6 +3894,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -4197,6 +4198,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5044883a49ca2..5850604f88207 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -136,6 +136,12 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
MT->getNumRows() * MT->getNumColumns(),
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(QT);
+ return Builder.getArrayType(convertType(MT->getElementType()),
+ MT->getNumRows() * MT->getNumColumns(),
+ ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ }
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
case Type::BitInt: {
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index f6bb43482c705..73d1753349fd4 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,6 +48,7 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
+ const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
>From fb104e6df911460b6bc2fa93d693cc8c6dee9efb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:20:46 -0400
Subject: [PATCH 06/43] Add LIT tests and fix minor build issues
---
clang/include/clang/AST/TypeBase.h | 8 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 94 +++++++++++++++
.../test/Preprocessor/coop-mat-opencl-ext.cl | 63 ++++++++++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 66 ++++++++++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 60 +++++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 83 +++++++++++++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 114 ++++++++++++++++++
clang/tools/libclang/CIndex.cpp | 1 +
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ++++++
9 files changed, 524 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen.cl
create mode 100644 clang/test/Preprocessor/coop-mat-opencl-ext.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema-neg.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-type-infra.cl
create mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 9b1c9ef646f1f..b814690461223 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4627,14 +4627,14 @@ class CooperativeMatrixType final : public MatrixType {
/// Return true if \p Scope is valid
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
}
/// Return true if \p Use is valid
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
}
/// Returns the maximum number of elements per dimension.
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
new file mode 100644
index 0000000000000..58b7a882fdc87
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -0,0 +1,94 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+//
+// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
+// SPIR-V intrinsic names.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
+// TargetExtType (visible in the call signature).
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// ---------------------------------------------------------------------------
+kernel void test_store(__global float *ptr, MatA_t a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+
+// ---------------------------------------------------------------------------
+// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a; MatB_t b; MatC_t c; MatC_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+
+// ---------------------------------------------------------------------------
+// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// ---------------------------------------------------------------------------
+kernel void test_binary_add(__global float *ptr) {
+ MatA_t a; MatA_t b; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a + b;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
+
+// ---------------------------------------------------------------------------
+// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// ---------------------------------------------------------------------------
+kernel void test_scalar_mul(__global float *ptr, float s) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a * s;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+
+// ---------------------------------------------------------------------------
+// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// ---------------------------------------------------------------------------
+kernel void test_unary_neg(__global float *ptr) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = -a;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_unary_neg
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarNeg
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
new file mode 100644
index 0000000000000..7181efbe7e2a8
--- /dev/null
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -0,0 +1,63 @@
+// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+//
+// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
+//
+// Tests: extension macro is predefined when enabled, extension can be
+// explicitly enabled/disabled via pragma, all four enum types and
+// their constants are visible under -finclude-default-header.
+
+// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E -dM %s \
+// RUN: | FileCheck %s --check-prefix=EXT
+
+// EXT: cl_ext_kernel_cooperative_matrix
+
+// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E %s \
+// RUN: | FileCheck %s --check-prefix=NOEXT
+
+// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+
+// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+void test_enum_constants(void) {
+ // coop_matrix_scope_t
+ coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
+ (void)s;
+
+ // coop_matrix_use_t
+ coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
+ coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
+ coop_matrix_use_t u2 = CLK_COOPERATIVE_MATRIX_ACCUMULATOR;
+ (void)u0; (void)u1; (void)u2;
+
+ // coop_matrix_layout_t
+ coop_matrix_layout_t l0 = CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR;
+ coop_matrix_layout_t l1 = CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR;
+ (void)l0; (void)l1;
+
+ // coop_matrix_operands_t
+ coop_matrix_operands_t op = CLK_COOPERATIVE_MATRIX_OPERAND_NONE;
+ (void)op;
+}
+
+// ── 2d. Enum constant values match the spec ──────────────────────────────────
+void test_enum_values(void) {
+ _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR == 0, "row major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR== 1, "col major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_OPERAND_NONE == 0, "operand none");
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
new file mode 100644
index 0000000000000..c6230eb94531f
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -0,0 +1,66 @@
+// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
+//
+// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
+// ExprConstant, ItaniumMangle, MicrosoftMangle,
+// ASTReader/ASTWriter TypeLoc serialisation.
+//
+// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=MANGLE
+//
+// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch \
+// RUN: -ast-dump %t.pch_probe.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+//
+// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
+kernel void test_mangling(MatA_t a) { (void)a; }
+
+// MANGLE: @{{.*}}test_mangling{{.*}}(
+
+// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
+// must still carry a coop_mat parameter type.
+// PCH: FunctionDecl {{.*}} pch_probe
+// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
+
+// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_structural_equiv(void) {
+ MatA_t *p = 0;
+ MatA_alias *q = p; // same canonical type
+ (void)q;
+}
+
+// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
+// integer, float, pointer …). Using __builtin_classify_type on it
+// compiles without error; result is 0 (no_type_class).
+void test_expr_constant(MatA_t a) {
+ int cls = __builtin_classify_type(a);
+ (void)cls;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
new file mode 100644
index 0000000000000..f8080448b6df5
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -0,0 +1,60 @@
+// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+//
+// Patch 4: Sema — negative tests for diagnostic paths.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 2. Invalid use value (99 is not 0/1/2)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 3. Mismatched element types in coop_mat_mulAdd
+// a/b are float matrices, c is an int matrix — should fire element type
+// mismatch diagnostic.
+// Use the two-step declare-then-assign pattern so that the coop_mat_load
+// fixup path fires correctly and we only get the intended mulAdd error.
+// ---------------------------------------------------------------------------
+kernel void test_muladd_type_mismatch(__global float *fptr,
+ __global int *iptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+ a = coop_mat_load(fptr, ROW_MAJOR, 16);
+ b = coop_mat_load(fptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+
+ MatC_int_t result;
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
+// This intentionally uses single-step initialization because the error
+// fires precisely when the LHS is NOT a coop mat type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_assignment(__global float *ptr) {
+ float bad;
+ bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
+ (void)bad;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
new file mode 100644
index 0000000000000..cec1f278b0037
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -0,0 +1,83 @@
+// clang/test/SemaOpenCL/coop-mat-sema.cl
+//
+// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
+// builtin validation, expr handling, TreeTransform.
+//
+// Valid code -- expected-no-diagnostics
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
+// match the LHS variable type. Declare first, then assign so that
+// Sema's IsCoopMatrixBuiltin path fires correctly.
+// ---------------------------------------------------------------------------
+kernel void test_load_store(__global float *ptr,
+ __global float *out_ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
+}
+
+// ---------------------------------------------------------------------------
+// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ MatC_t result;
+ result = coop_mat_mulAdd(a, b, c);
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4d. Binary element-wise operators (+, -)
+// ---------------------------------------------------------------------------
+void test_binary_ops(MatA_t a, MatA_t b) {
+ MatA_t r_add = a + b;
+ MatA_t r_sub = a - b;
+ (void)r_add; (void)r_sub;
+}
+
+// ---------------------------------------------------------------------------
+// 4e. Scalar multiply operator
+// ---------------------------------------------------------------------------
+void test_scalar_ops(MatA_t a, float s) {
+ MatA_t r = a * s;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4f. Unary minus
+// ---------------------------------------------------------------------------
+void test_unary_minus(MatA_t a) {
+ MatA_t r = -a;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
+// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// ---------------------------------------------------------------------------
+kernel void test_assignment(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
new file mode 100644
index 0000000000000..2e6700973e611
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -0,0 +1,114 @@
+// clang/test/SemaOpenCL/coop_mat_type_infra.cl
+//
+// Patch 1: CooperativeMatrixType AST node and type infrastructure.
+//
+// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
+// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
+// RecursiveASTVisitor traversal, mergeTypes compatibility,
+// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -ast-dump %s \
+// RUN: | FileCheck %s --check-prefix=AST
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+
+// ---------------------------------------------------------------------------
+// Enum constants (from opencl-c-base.h via -finclude-default-header)
+// ---------------------------------------------------------------------------
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+// ---------------------------------------------------------------------------
+// 1. Basic type construction — four use roles, float element type.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
+typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
+
+// AST: TypedefDecl {{.*}} MatA_float16x16
+// AST: TypedefDecl {{.*}} MatB_float16x16
+// AST: TypedefDecl {{.*}} MatC_float16x16
+// AST: TypedefDecl {{.*}} MatA_int8x8
+
+// ---------------------------------------------------------------------------
+// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// ---------------------------------------------------------------------------
+void test_type_spelling(void) {
+ MatA_float16x16 a;
+ MatB_float16x16 b;
+ MatC_float16x16 c;
+ MatA_int8x8 d;
+}
+
+// AST: VarDecl {{.*}} a {{.*}}coop_mat(
+// AST: VarDecl {{.*}} b {{.*}}coop_mat(
+// AST: VarDecl {{.*}} c {{.*}}coop_mat(
+// AST: VarDecl {{.*}} d {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
+// float(4B)*16*16 = 1024 B = 8192 bits
+// int(4B)*8*8 = 256 B = 2048 bits
+// ---------------------------------------------------------------------------
+void test_sizeof(void) {
+ _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
+ _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
+}
+
+// ---------------------------------------------------------------------------
+// 4. Parameter / return type — type preserved through function boundary.
+// ---------------------------------------------------------------------------
+MatA_float16x16 test_param_return(MatA_float16x16 in) {
+ return in;
+}
+
+// AST: FunctionDecl {{.*}} test_param_return
+// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 5. TypeLoc operand traversal — all four operand slots populated.
+// ---------------------------------------------------------------------------
+void test_typeloc_operands(void) {
+ float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
+ (void)local_acc;
+}
+
+// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// ---------------------------------------------------------------------------
+void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
+ (void)x;
+}
+
+// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_merge_types(void) {
+ MatA_float16x16 *p = 0;
+ MatA_alias *q = p; // same canonical type — no diagnostic
+ (void)q;
+}
+
+// ---------------------------------------------------------------------------
+// 8. PCH serialisation round-trip.
+// ---------------------------------------------------------------------------
+// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8d246d41f31f6..951fedea44ee4 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,6 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
new file mode 100644
index 0000000000000..c4e206b76b760
--- /dev/null
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -0,0 +1,39 @@
+; llvm/test/Verifier/coop_mat_verifier_fix.ll
+;
+; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
+;
+; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
+; ((A && B) <= 64). After the fix the parentheses are explicit.
+;
+; This test verifies that a valid @llvm.experimental.noalias.scope.decl
+; call with a 2-element scope list (no third argument) passes the verifier
+; without error, and that an invalid third argument (> 64 bits) is caught.
+;
+; ── 6a. Valid call — no third argument → verifier must accept ────────────────
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
+; RUN: llvm-as < %s | opt -passes=verify -disable-output
+;
+; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
+; RUN: llvm-as %s -o %t.bc
+; RUN: opt -passes=verify %t.bc -disable-output
+
+; VALID: @llvm.experimental.noalias.scope.decl
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+define void @test_no_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+; Third arg present and <= 64 bits wide — must also pass
+define void @test_valid_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain_a"}
>From 36e791e0c9644be16c595087042fbdec3c7e742c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 18:39:15 -0400
Subject: [PATCH 07/43] Fix formatting issues
---
clang/include/clang/Sema/Sema.h | 15 +++---
clang/lib/CodeGen/CGBuiltin.cpp | 76 +++++++++++++++------------
clang/lib/CodeGen/CGExpr.cpp | 63 ++++++++++------------
clang/lib/CodeGen/CGExprScalar.cpp | 12 ++---
clang/lib/Sema/SemaChecking.cpp | 39 +++++++-------
clang/lib/Sema/SemaExpr.cpp | 35 ++++++------
clang/lib/Sema/TreeTransform.h | 2 +-
clang/lib/Serialization/ASTReader.cpp | 3 +-
clang/lib/Serialization/ASTWriter.cpp | 3 +-
9 files changed, 120 insertions(+), 128 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 0847e05509cb4..bbaeb6ae37240 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3114,19 +3114,16 @@ class Sema final : public SemaBase {
// Coop matrix handling.
void CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
+ SourceLocation MatrixLoc, CallExpr *call);
void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
+ QualType BufferType,
+ SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
- ExprResult CallResult);
- ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
- ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 6156212d20a66..6f36cfa5b9b38 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2797,7 +2797,7 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
llvm::Type *Tys[] = {ElTy};
// Unsigned arguments for TargetExtType
unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
+ MTy->getUse()};
// Create a TargetExtType to represent the coop matrix type
llvm::TargetExtType *RetType = llvm::TargetExtType::get(
CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
@@ -4677,8 +4677,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_load without coop_mat output operand");
+ CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Layout = EmitScalarExpr(E->getArg(1));
@@ -4692,7 +4691,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LoadFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
@@ -4709,8 +4709,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_store without coop_mat input operand");
+ CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Arg0 = EmitScalarExpr(E->getArg(1));
@@ -4720,11 +4719,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
- VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
+ false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee StoreFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
@@ -4735,8 +4736,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_mulAdd: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_mulAdd without coop_mat output operand");
+ CGM.ErrorUnsupported(E,
+ "coop_mat_mulAdd without coop_mat output operand");
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
@@ -4760,11 +4761,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto *AType = getTargetExtType(*this, CGM, MATy);
auto *BType = getTargetExtType(*this, CGM, MBTy);
auto *CType = getTargetExtType(*this, CGM, MCTy);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee MatMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
@@ -4778,7 +4780,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_binary_div: {
auto Arg0 = EmitScalarExpr(E->getArg(0));
auto Arg1 = EmitScalarExpr(E->getArg(1));
-
+
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
@@ -4795,13 +4797,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
-
- std::string Name = (ElTy->isIntegerTy())
- ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
- : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
- llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name =
+ (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
@@ -4827,10 +4831,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
@@ -4854,10 +4859,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarNegFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
@@ -4870,10 +4876,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Init = EmitScalarExpr(E->getArg(0));
auto *RetType = getTargetExtType(*this, CGM, MTy);
// Set function type.
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {Init->getType()}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {Init->getType()}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee InitFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(InitFn, {Init});
@@ -4885,11 +4892,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
auto *AType = getTargetExtType(*this, CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- ConvertType(E->getType()), {AType}, false);
-
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
+
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LengthFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 31251fdc3535c..45c1bf6564514 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7695,44 +7695,37 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
- llvm::Value *ScalarVal,
- QualType CoopMatQTy)
-{
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
-
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(
- VecTy->getElementCount(), C);
-
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(
- VecTy->getElementCount(),
- ScalarVal,
- "coopmat.broadcast");
-}
-
-llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
- BinaryOperatorKind Opcode,
- llvm::Value *LHS,
- llvm::Value *RHS,
- QualType ResultTy)
-{
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy) {
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy) {
auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
switch (Opcode) {
case BO_Add:
return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+ : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
case BO_Sub:
return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+ : Builder.CreateSub(LHS, RHS, "coopmat.isub");
case BO_Mul:
// mat * mat → element-wise FMul/IMul
@@ -7741,14 +7734,16 @@ llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
// mat * scalar: broadcast scalar then element-wise mul
llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
}
return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+ : Builder.CreateMul(LHS, RHS, "coopmat.imul");
case BO_Div:
- if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ if (IsFloat)
+ return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned)
+ return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
default:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 4802bdff0f805..354143adf5c02 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -957,10 +957,9 @@ class ScalarExprEmitter
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
if (E->getType()->isCooperativeMatrixType()) { \
- return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
- CGF.EmitScalarExpr(E->getLHS()), \
- CGF.EmitScalarExpr(E->getRHS()), \
- E->getType()); \
+ return CGF.EmitCoopMatBinaryOp( \
+ E->getOpcode(), CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), E->getType()); \
} \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
@@ -3692,11 +3691,12 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ QualType CompTy =
+ E->getType()->getAs<CooperativeMatrixType>()->getElementType();
if (CompTy->isFloatingType())
return Builder.CreateFNeg(Val, "coopmat.fneg");
else
- return Builder.CreateNeg(Val, "coopmat.ineg");
+ return Builder.CreateNeg(Val, "coopmat.ineg");
}
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 6a74740dec507..85b5e8aba592f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17443,8 +17443,7 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
}
// Check coop_mat_load/store buffer pointer.
-bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
- unsigned PtrArgIdx) {
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
bool ArgError = false;
Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
@@ -17475,8 +17474,8 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
// Check coop_mat_load/store matrix element has same type with buffer pointer.
void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
@@ -17492,8 +17491,8 @@ void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
}
void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *call) {
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
FunctionDecl *F = call->getDirectCallee();
assert(F);
@@ -17502,7 +17501,7 @@ void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
assert(Fname);
if (Fname->isStr("coop_mat_load"))
CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ MatrixLoc);
}
// Check coop_mat_load/store layout argument
@@ -17526,7 +17525,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
}
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
@@ -17545,7 +17544,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
return CallResult;
@@ -17584,7 +17583,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
- SourceLocation BLoc) {
+ SourceLocation BLoc) {
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
@@ -17613,7 +17612,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
}
ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17621,7 +17620,7 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ Arg1->getBeginLoc());
TheCall->setType(Arg0->getType());
@@ -17637,7 +17636,7 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
}
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
@@ -17685,7 +17684,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17696,7 +17695,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
return ExprError();
@@ -17729,17 +17728,17 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// matrix type.
if (ConstMType) {
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
}
if (CoopMType) {
QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(),CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
+ CoopMType->getElementType(), CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 78bbb3768e725..33e9f21101b77 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -11408,8 +11408,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
if (LHSTy->isSveVLSBuiltinType() || RHSTy->isSveVLSBuiltinType())
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
- if (!IsDiv &&
- (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
+ if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
@@ -14040,10 +14039,11 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return QualType();
if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
+ RHS.get()->getType()->isCooperativeMatrixType()) {
auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a matrix");
if (LHSMatType && RHSMatType) {
if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
return InvalidOperands(Loc, LHS, RHS);
@@ -14054,14 +14054,14 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
RHS.get()->getType().getUnqualifiedType());
QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
+ RHSELTy = RHSMatType->getElementType();
if (!Context.hasSameType(LHSELTy, RHSELTy))
return InvalidOperands(Loc, LHS, RHS);
return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
- LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
- LHSMatType->getUse());
+ Context.getCommonSugaredType(LHSELTy, RHSELTy),
+ LHSMatType->getScope(), LHSMatType->getNumRows(),
+ RHSMatType->getNumColumns(), LHSMatType->getUse());
}
}
@@ -15803,17 +15803,13 @@ ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Add:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add, Args);
case BO_Sub:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub, Args);
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul, Args);
case BO_Div:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div, Args);
default:
break;
}
@@ -15828,8 +15824,7 @@ ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul, Args);
default:
break;
}
@@ -15877,7 +15872,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
- RHSExpr->getBeginLoc()))
+ RHSExpr->getBeginLoc()))
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
@@ -15919,7 +15914,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
assert(call);
call->setType(LHSExpr->getType());
CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
+ call);
CheckCoopMatrixMatMulOutput(call);
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 498551937b85d..eb7af59cc0fac 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1053,7 +1053,7 @@ class TreeTransform {
QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
unsigned NumRows, unsigned NumColumns,
unsigned Use);
-
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index ee39d6b883cff..759176f58e46e 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7672,8 +7672,7 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
-void TypeLocReader::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
TL.setAttrOperandParensRange(readSourceRange());
TL.setAttrScopeOperand(Reader.readExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 835ee0ac49071..dcbabd39ad8f0 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -457,8 +457,7 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
-void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
SourceRange range = TL.getAttrOperandParensRange();
addSourceLocation(range.getBegin());
>From 656fc37e6ca9b81e648def6a152b5c9fd7803959 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 15 Jul 2026 12:12:32 -0400
Subject: [PATCH 08/43] Add tests for coop_mat_init and coop_mat_length
---
.../coop-mat-init-length-codegen.cl | 112 ++++++++++++++++++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 94 +++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
new file mode 100644
index 0000000000000..b18f5617338c8
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -0,0 +1,112 @@
+// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+// ---------------------------------------------------------------------------
+// Type definitions.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init — float scalar.
+// Expected IR: call __spirv_CompositeConstruct with the scalar value,
+// returning a spirv.CooperativeMatrixKHR TargetExtType.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_float
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float
+
+kernel void test_init_float(global float *out) {
+ MatA_t a;
+ a = coop_mat_init(1.0f);
+ (void)a;
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init — integer scalar.
+// Expected IR: call __spirv_CompositeConstruct with an i32 value.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_int
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(i32
+
+kernel void test_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+ (void)m;
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init — zero initialisation.
+// Expected IR: call __spirv_CompositeConstruct with 0.0.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_zero
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+
+kernel void test_init_zero(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ (void)acc;
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length — returns unsigned int.
+// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
+// an i32 alloca.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+
+kernel void test_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ *out = len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_init then coop_mat_length — combined flow.
+// Verifies the TargetExtType produced by init flows into length unchanged.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_then_length
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_init_then_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(2.0f);
+ *out = coop_mat_length(a);
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// The call must NOT be a plain 'call' — it must be 'call spir_func'.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length_cc
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_length_cc(global unsigned int *out) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ *out = coop_mat_length(acc);
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
new file mode 100644
index 0000000000000..ab499bc1b4af7
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -0,0 +1,94 @@
+// Tests for coop_mat_init and coop_mat_length — Sema (positive).
+//
+// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
+// so arg-count and arg-type validation are not performed by Sema for these two
+// builtins. Only the positive (valid-code) path and the one assignment-target
+// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
+// tested here.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics (sections 1-6 must compile cleanly)
+
+// ---------------------------------------------------------------------------
+// Type definitions shared across all tests.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_B))) MatB_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init: initialise each use role with a float scalar.
+// Two-step (declare then assign) is required because the return type is
+// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_basic(void) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_init(1.0f);
+ b = coop_mat_init(2.0f);
+ c = coop_mat_init(0.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init: integer element type.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_chained(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ acc = coop_mat_init(1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length: basic usage — returns unsigned int directly.
+// No two-step required for the length call itself.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_basic(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ (void)len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_length: all three use roles.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_roles(void) {
+ MatA_t a; a = coop_mat_init(0.0f);
+ MatB_t b; b = coop_mat_init(0.0f);
+ MatC_t c; c = coop_mat_init(0.0f);
+ unsigned int la = coop_mat_length(a);
+ unsigned int lb = coop_mat_length(b);
+ unsigned int lc = coop_mat_length(c);
+ (void)la; (void)lb; (void)lc;
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length: result used in arithmetic.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_arith(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int half_len = coop_mat_length(a) / 2u;
+ (void)half_len;
+}
>From 9158617a7e570e345998c897990e5189ed469b57 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 19:21:14 -0400
Subject: [PATCH 09/43] Add mangling support
Signed-off-by: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
---
clang/lib/CodeGen/CGBuiltin.cpp | 91 ++++++++-
.../coop-mat-codegen-mangling.cl | 174 ++++++++++++++++++
.../coop-mat-init-length-codegen.cl | 14 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 15 ++
4 files changed, 280 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 6f36cfa5b9b38..71fcca9f2fcd6 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2805,6 +2805,62 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
return RetType;
}
+/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
+/// TargetExtType: element type + scope + rows + cols + use.
+/// E.g. target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) ->
+/// "_f32_sc3_16x16_u2"
+static std::string getCoopMatFullSuffix(llvm::Type *Ty) {
+ auto *TET = cast<llvm::TargetExtType>(Ty);
+ llvm::Type *ElemTy = TET->getTypeParameter(0);
+
+ // Element type string
+ std::string ElemStr;
+ if (ElemTy->isFloatTy())
+ ElemStr = "f32";
+ else if (ElemTy->isHalfTy())
+ ElemStr = "f16";
+ else if (ElemTy->isDoubleTy())
+ ElemStr = "f64";
+ else if (auto *IntTy = dyn_cast<llvm::IntegerType>(ElemTy))
+ ElemStr = "i" + std::to_string(IntTy->getBitWidth());
+ else
+ llvm_unreachable("Unsupported cooperative matrix element type");
+
+ // Integer parameters: scope, rows, cols, use
+ unsigned Scope = TET->getIntParameter(0);
+ unsigned Rows = TET->getIntParameter(1);
+ unsigned Cols = TET->getIntParameter(2);
+ unsigned Use = TET->getIntParameter(3);
+
+ return "_" + ElemStr + "_sc" + std::to_string(Scope) + "_" +
+ std::to_string(Rows) + "x" + std::to_string(Cols) + "_u" +
+ std::to_string(Use);
+}
+
+/// Returns a name suffix encoding the address space of a pointer type,
+/// matching OpenCL/SPIR-V address space conventions.
+/// E.g. ptr addrspace(1) -> "_global"
+/// ptr addrspace(3) -> "_local"
+/// ptr addrspace(0) -> "_private"
+/// ptr addrspace(4) -> "_generic"
+static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
+ unsigned AS = cast<llvm::PointerType>(PtrTy)->getAddressSpace();
+ switch (AS) {
+ case 0:
+ return "_private";
+ case 1:
+ return "_global";
+ case 2:
+ return "_constant";
+ case 3:
+ return "_local";
+ case 4:
+ return "_generic";
+ default:
+ return "_as" + std::to_string(AS);
+ }
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4689,7 +4745,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy = llvm::FunctionType::get(
RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee LoadFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4722,7 +4781,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
llvm::FunctionCallee StoreFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4764,7 +4826,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(Arg0->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg1->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg2->getType()));
llvm::FunctionCallee MatMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
@@ -4804,6 +4870,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
(ElTy->isIntegerTy())
? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
: getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ // Function name mangling.
+ Name = Name + getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee BinaryFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
@@ -4833,7 +4901,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
@@ -4861,7 +4931,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarNegFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
@@ -4878,7 +4950,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee InitFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
@@ -4895,7 +4969,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(0)->getType()));
llvm::FunctionCallee LengthFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
new file mode 100644
index 0000000000000..9fec09287f890
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -0,0 +1,174 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+//
+// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
+// Covers three mangling axes:
+// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
+// (B) coop_mat_mulAdd -- different element types (float vs int)
+// (C) coop_mat_load -- different pointer address spaces
+// (D) coop_mat_store -- different pointer address spaces
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
+// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
+// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+// ---------------------------------------------------------------------------
+// Matrix type aliases used across tests
+// ---------------------------------------------------------------------------
+
+// --- float 16x16 (MulAdd group 1) ---
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatF_16x16_C;
+
+// --- float 8x32 / 32x8 / 8x8 (MulAdd group 2 -- same elem type, diff dims) ---
+typedef float __attribute__((coop_mat(SCOPE, 8, 32, USE_A))) MatF_8x32_A;
+typedef float __attribute__((coop_mat(SCOPE, 32, 8, USE_B))) MatF_32x8_B;
+typedef float __attribute__((coop_mat(SCOPE, 8, 8, USE_C))) MatF_8x8_C;
+
+// --- int 16x16 (MulAdd group 3 -- different element type to group 1) ---
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
+
+// ===========================================================================
+// (A) MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
+// ===========================================================================
+
+kernel void test_muladd_f32_16x16(__global float *ptr) {
+ MatF_16x16_A a;
+ MatF_16x16_B b;
+ MatF_16x16_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+kernel void test_muladd_f32_8x32(__global float *ptr) {
+ MatF_8x32_A a;
+ MatF_32x8_B b;
+ MatF_8x8_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 8);
+ b = coop_mat_load(ptr, ROW_MAJOR, 32);
+ c = coop_mat_load(ptr, ROW_MAJOR, 8);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_8x32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x32_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
+
+// ===========================================================================
+// (B) MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
+// ===========================================================================
+
+kernel void test_muladd_i32_16x16(__global int *iptr) {
+ MatI_16x16_A a;
+ MatI_16x16_B b;
+ MatI_16x16_C c, r;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16);
+ b = coop_mat_load(iptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_i32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+// ===========================================================================
+// (C) Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
+// ===========================================================================
+
+kernel void test_load_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_load_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (D) Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
+// ===========================================================================
+
+kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (E) Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
+// ===========================================================================
+
+kernel void test_no_cross_contamination(__global float *gptr,
+ __local float *lptr) {
+ MatF_16x16_A ag, al;
+ ag = coop_mat_load(gptr, ROW_MAJOR, 16);
+ al = coop_mat_load(lptr, ROW_MAJOR, 16);
+ (void)ag; (void)al;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_no_cross_contamination
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_global_f32_{{.*}}addrspace(3)
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
+
+// Verify distinct declare lines exist (these appear at end of IR).
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b18f5617338c8..8dd1aab2b2a13 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -26,7 +26,7 @@ typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_float(global float *out) {
MatA_t a;
@@ -41,7 +41,7 @@ kernel void test_init_float(global float *out) {
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(i32
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_int(void) {
MatA_int_t m;
@@ -56,7 +56,7 @@ kernel void test_init_int(void) {
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+// CHECK-SAME: @__spirv_CompositeConstruct_{{.*}}(float {{.*}}0
kernel void test_init_zero(void) {
MatC_t acc;
@@ -71,7 +71,7 @@ kernel void test_init_zero(void) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
kernel void test_length(global unsigned int *out) {
@@ -88,8 +88,8 @@ kernel void test_length(global unsigned int *out) {
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: @__spirv_CompositeConstruct
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_init_then_length(global unsigned int *out) {
MatA_t a;
@@ -103,7 +103,7 @@ kernel void test_init_then_length(global unsigned int *out) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_length_cc(global unsigned int *out) {
MatC_t acc;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 800f3a4d0c4df..383f72d4e7c16 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -569,6 +569,21 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
}
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
+ // Cooperative-matrix builtins emitted by Clang CodeGen carry mangling
+ // suffixes to avoid IR function-signature collisions, e.g.:
+ // "__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global"
+ // Strip everything from the first '_' after "__spirv_CooperativeMatrix"
+ // so the name matches the canonical DemangledNativeBuiltin entry in
+ // SPIRVBuiltins.td (e.g. "__spirv_CooperativeMatrixLoadKHR").
+ static constexpr StringRef CoopMatPrefix = "__spirv_CooperativeMatrix";
+ if (Name.starts_with(CoopMatPrefix)) {
+ size_t SuffixPos = Name.find('_', CoopMatPrefix.size());
+ if (SuffixPos != StringRef::npos)
+ return Name.substr(0, SuffixPos).str();
+ // No mangling suffix present — return the name as-is.
+ return Name.str();
+ }
+
bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
>From 9f6ea0fe2a7b5504ea3bdce1d3006d7a4e7b36e5 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 13:23:45 -0400
Subject: [PATCH 10/43] Change name of extension to cl_khr_cooperative_matrix
---
clang/include/clang/Basic/Builtins.td | 2 +-
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
.../test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../CodeGenOpenCL/coop-mat-init-length-codegen.cl | 2 +-
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 12 ++++++------
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-init-length-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 6 +++---
11 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ab73570c9bbcb..66c214440a99f 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,7 +1931,7 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
-// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+// Cooperative Matrix builtins for cl_khr_cooperative_matrix
def CoopMatLoad : Builtin {
let Spellings = ["coop_mat_load"];
let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 89e96dca85a59..8a583ad504c70 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9fec09287f890..9d55bd8c61cc3 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -8,7 +8,7 @@
// (D) coop_mat_store -- different pointer address spaces
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 58b7a882fdc87..ad8df2939269f 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -4,7 +4,7 @@
// SPIR-V intrinsic names.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 8dd1aab2b2a13..dfa763b060e72 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -1,7 +1,7 @@
// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 7181efbe7e2a8..f77fe542fd210 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,6 +1,6 @@
// clang/test/Preprocessor/coop_mat_opencl_ext.cl
//
-// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// Patch 2: cl_khr_cooperative_matrix registration in
// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
//
// Tests: extension macro is predefined when enabled, extension can be
@@ -9,23 +9,23 @@
// ── 2a. Extension macro is predefined when the extension is enabled ─────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
// RUN: | FileCheck %s --check-prefix=EXT
-// EXT: cl_ext_kernel_cooperative_matrix
+// EXT: cl_khr_cooperative_matrix
// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
// RUN: | FileCheck %s --check-prefix=NOEXT
-// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+// NOEXT-NOT: cl_khr_cooperative_matrix
// ── 2c. Enum constants are visible when extension is enabled ─────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index c6230eb94531f..e8fad815806ec 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -6,24 +6,24 @@
//
// ── 3a. Itanium mangling ────────────────────────────────────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch \
// RUN: -ast-dump %t.pch_probe.cl \
// RUN: | FileCheck %s --check-prefix=PCH
//
// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index ab499bc1b4af7..799f7a9e15a35 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -7,7 +7,7 @@
// tested here.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics (sections 1-6 must compile cleanly)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index f8080448b6df5..0c27978d492a9 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -3,7 +3,7 @@
// Patch 4: Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index cec1f278b0037..bcedd94fdbf17 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -5,7 +5,7 @@
//
// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 2e6700973e611..43adeebd94500 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -8,16 +8,16 @@
// TypeLoc operand slots, sizeof / getTypeInfoImpl.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -ast-dump %s \
// RUN: | FileCheck %s --check-prefix=AST
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
>From bfc50469090fbaaf8bffcd24dde434245d030095 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 14:37:44 -0400
Subject: [PATCH 11/43] Fix test fails
---
clang/lib/Sema/SemaType.cpp | 6 ++++++
clang/test/AST/undocumented-attrs.cpp | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 4dcf2c6902501..476f4d24fc1ec 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9044,6 +9044,12 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Scope, Use, /* IsCoopMat */ true);
if (!T.isNull())
CurType = T;
+ } else {
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
}
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index 525466b1efdc0..dff9d3845062a 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,6 +27,7 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
+CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -88,4 +89,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
+CHECK-NEXT: Total: 83
>From e377f66b8f8e0fd2038acd085d24aa4bcd0ca650 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sun, 9 Aug 2026 00:57:18 -0400
Subject: [PATCH 12/43] Fix failing lldb build and a couple of tests
---
.../test/Misc/pragma-attribute-supported-attributes-list.test | 1 +
lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 3 +++
llvm/test/Verifier/coop-mat-verifier-fix.ll | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 8f8c1aa3d3f9c..c906921cc1b7a 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -63,6 +63,7 @@
// CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
// CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
// CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CoopMatrixType (SubjectMatchRule_type_alias, SubjectMatchRule_variable, SubjectMatchRule_variable_is_parameter, SubjectMatchRule_field)
// CHECK-NEXT: CoroAwaitElidable (SubjectMatchRule_record)
// CHECK-NEXT: CoroAwaitElidableArgument (SubjectMatchRule_variable_is_parameter)
// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function)
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index f3e8baaf38453..9d2f337e856ac 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -4224,6 +4224,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
// Matrix types that we're not sure how to display at the moment.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5101,6 +5102,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type) {
break;
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5274,6 +5276,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
// Matrix types we're not sure how to display yet.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
index c4e206b76b760..7f2eedb657d41 100644
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -22,14 +22,14 @@
declare void @llvm.experimental.noalias.scope.decl(metadata)
define void @test_no_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
; Third arg present and <= 64 bits wide — must also pass
define void @test_valid_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
>From 03fe6d2442b05e12ec35c5dd6a7934e2489220b6 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 4 Sep 2026 18:58:24 -0400
Subject: [PATCH 13/43] Minor changes to sync with latest upstream changes
---
clang/lib/AST/ASTContext.cpp | 10 ++++------
clang/lib/CodeGen/CGBuiltin.cpp | 2 --
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 6052ee26e96fb..a7e096a4339cf 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4957,9 +4957,8 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
- void *InsertPos = nullptr;
- if (CooperativeMatrixType *MTP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ llvm::FoldingSetInsertToken Token;
+ if (CooperativeMatrixType *MTP = CooperativeMatrixTypes.lookup(ID, Token))
return QualType(MTP, 0);
QualType Canonical;
@@ -4967,15 +4966,14 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
NumRows, NumColumns, Use);
- CooperativeMatrixType *NewIP =
- CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ CooperativeMatrixType *NewIP = CooperativeMatrixTypes.lookup(ID, Token);
assert(!NewIP && "Matrix type shouldn't already exist in the map");
(void)NewIP;
}
auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
- CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ CooperativeMatrixTypes.insert(New, Token);
Types.push_back(New);
return QualType(New, 0);
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 71fcca9f2fcd6..b6d79bd240f06 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2861,8 +2861,6 @@ static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
}
}
-} // namespace
-
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const CallExpr *E,
ReturnValueSlot ReturnValue) {
>From e20d4d8ac9e037ab38b460572d65e1dbb1a2a29c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 13:55:14 -0400
Subject: [PATCH 14/43] Update CooperativeMatrixType class definition to be
derived directly from Type class, instead of MatrixType class.
---
clang/include/clang/AST/OperationKinds.def | 3 +
clang/include/clang/AST/TypeBase.h | 78 ++++++---
clang/include/clang/AST/TypeLoc.h | 105 +++++++----
clang/include/clang/AST/TypeProperties.td | 3 +
clang/include/clang/Basic/TypeNodes.td | 2 +-
clang/include/clang/CIR/Dialect/IR/CIROps.td | 1 +
clang/include/clang/Sema/Sema.h | 27 ++-
clang/lib/AST/ASTContext.cpp | 18 +-
clang/lib/AST/Expr.cpp | 1 +
clang/lib/AST/ExprConstant.cpp | 2 +
clang/lib/AST/Type.cpp | 16 --
clang/lib/CIR/CodeGen/CIRGenExpr.cpp | 2 +
clang/lib/CodeGen/CGExpr.cpp | 78 +++++----
clang/lib/CodeGen/CGExprAgg.cpp | 2 +
clang/lib/CodeGen/CGExprComplex.cpp | 1 +
clang/lib/CodeGen/CGExprConstant.cpp | 1 +
clang/lib/CodeGen/CGExprScalar.cpp | 16 +-
clang/lib/CodeGen/CodeGenFunction.h | 23 ++-
clang/lib/CodeGen/CodeGenTBAA.cpp | 5 +-
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +-
clang/lib/CodeGen/QualTypeMapper.h | 1 -
clang/lib/Edit/RewriteObjCFoundationAPI.cpp | 1 +
clang/lib/Sema/SemaCast.cpp | 16 ++
clang/lib/Sema/SemaChecking.cpp | 32 ++--
clang/lib/Sema/SemaDecl.cpp | 2 +-
clang/lib/Sema/SemaExpr.cpp | 175 ++++++++++++++-----
clang/lib/Sema/SemaTemplateDeduction.cpp | 8 +-
clang/lib/Sema/SemaType.cpp | 145 ++++++++++-----
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 26 +--
clang/tools/libclang/CIndex.cpp | 2 +-
30 files changed, 521 insertions(+), 277 deletions(-)
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index 8a13ad988403b..d0cda0551f2d5 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -185,6 +185,9 @@ CAST_OPERATION(ToVoid)
/// CK_MatrixCast - A cast between matrix types of the same dimensions.
CAST_OPERATION(MatrixCast)
+/// CK_CoopMatrixCast - A cast between compatible cooperative matrix types.
+CAST_OPERATION(CoopMatrixCast)
+
/// CK_VectorSplat - A conversion from an arithmetic type to a
/// vector of that element type. Fills all elements ("splats") with
/// the source value.
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b814690461223..0b83ac3fd03e4 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4483,7 +4483,6 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
- T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4579,65 +4578,81 @@ class ConstantMatrixType final : public MatrixType {
}
};
-/// Represents a cooperative matrix type.
-class CooperativeMatrixType final : public MatrixType {
+/// Represents an opaque OpenCL cooperative matrix type.
+///
+/// Unlike MatrixType, a cooperative matrix is not an ordinary matrix value.
+/// It is an opaque, distributed object whose shape and role are part of its
+/// type identity.
+class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
protected:
friend class ASTContext;
+ /// Element type of the cooperative matrix.
+ QualType ElementType;
+
/// Number of rows and columns.
unsigned NumRows;
unsigned NumColumns;
- /// Scope and use
+ /// Cooperative matrix scope and use.
unsigned Scope;
unsigned Use;
static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
- CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
- unsigned NRows, unsigned NColumns, unsigned Use,
- QualType CanonElementType);
+ CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
+ unsigned NumColumns, unsigned Use,
+ QualType CanonicalType)
+ : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
+ NumRows, NumColumns, Use, CanonicalType) {}
- CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
- unsigned Scope, unsigned NRows, unsigned NColumns,
- unsigned Use, QualType CanonElementType);
+ CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use, QualType CanonicalType)
+ : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
+ Scope(Scope), Use(Use) {}
public:
- /// Returns the number of rows in the matrix.
+ /// Returns the element type.
+ QualType getElementType() const { return ElementType; }
+
+ /// Returns the number of rows.
unsigned getNumRows() const { return NumRows; }
- /// Returns the number of columns in the matrix.
+ /// Returns the number of columns.
unsigned getNumColumns() const { return NumColumns; }
- /// Returns the scope of the matrix.
+ /// Returns the cooperative matrix scope.
unsigned getScope() const { return Scope; }
- /// Returns the use of the matrix.
+ /// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into a vector.
+ /// Returns the number of elements required to embed the matrix into
+ /// a vector representation.
unsigned getNumElementsFlattened() const {
return getNumRows() * getNumColumns();
}
- /// Returns true if \p NumElements is a valid matrix dimension.
+ /// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
}
- /// Return true if \p Scope is valid
+ /// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
+ return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
}
- /// Return true if \p Use is valid
+ /// Returns true if \p Use is a valid cooperative matrix use.
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
+ return Use == 0 || // CLK_COOPERATIVE_MATRIX_A
+ Use == 1 || // CLK_COOPERATIVE_MATRIX_B
+ Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum number of elements per dimension.
+ /// Returns the maximum valid number of elements per dimension.
static constexpr unsigned getMaxElementsPerDimension() {
return MaxElementsPerDimension;
}
@@ -4658,6 +4673,23 @@ class CooperativeMatrixType final : public MatrixType {
ID.AddInteger(TypeClass);
}
+ static bool isValidElementType(QualType ElemTy) {
+ return ElemTy->isSpecificBuiltinType(BuiltinType::Char_S) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UChar) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Short) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UShort) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Int) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::UInt) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Long) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::ULong) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Half) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Float) ||
+ ElemTy->isSpecificBuiltinType(BuiltinType::Double);
+ }
+
+ bool isSugared() const { return false; }
+ QualType desugar() const { return QualType(this, 0); }
+
static bool classof(const Type *T) {
return T->getTypeClass() == CooperativeMatrix;
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 24fcda8f4c7e5..122090f7e6ec5 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,10 +2142,8 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
- Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
- Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2153,50 +2151,26 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
+ /// ^~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
- void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
-
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
- /// The attribute's scope operand (only for cooperative matrix).
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^
- Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
- void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
-
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
- /// For cooperative matrix, it is
- /// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2213,10 +2187,8 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
- setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
- setAttrUseOperand(nullptr);
}
};
@@ -2224,15 +2196,82 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
-class CooperativeMatrixTypeLoc
- : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
- CooperativeMatrixType> {};
-
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
DependentSizedMatrixType> {};
+struct CooperativeMatrixTypeLocInfo {
+ SourceLocation AttrLoc;
+ SourceRange OperandParens;
+ Expr *ScopeOperand;
+ Expr *RowOperand;
+ Expr *ColumnOperand;
+ Expr *UseOperand;
+};
+
+class CooperativeMatrixTypeLoc
+ : public ConcreteTypeLoc<UnqualTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType,
+ CooperativeMatrixTypeLocInfo> {
+public:
+ /// The location of the attribute name, i.e.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^^~~~~~~~~~~~~~~~~~~
+ SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
+ void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+
+ /// The attribute's scope operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
+ /// The attribute's row operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
+ void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
+
+ /// The attribute's column operand, if it has one.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
+ void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+
+ /// The attribute's use operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
+ /// The location of the parentheses around the operand, if there is
+ /// an operand.
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^
+ SourceRange getAttrOperandParensRange() const {
+ return getLocalData()->OperandParens;
+ }
+ void setAttrOperandParensRange(SourceRange range) {
+ getLocalData()->OperandParens = range;
+ }
+
+ SourceRange getLocalSourceRange() const {
+ SourceRange range(getAttrNameLoc());
+ range.setEnd(getAttrOperandParensRange().getEnd());
+ return range;
+ }
+
+ void initializeLocal(ASTContext &Context, SourceLocation loc) {
+ setAttrNameLoc(loc);
+ setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
+ setAttrRowOperand(nullptr);
+ setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
+ }
+};
+
// FIXME: location of the '_Complex' keyword.
class ComplexTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
ComplexTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 88d84d6d3234f..d134edc39e92c 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -264,6 +264,9 @@ let Class = ConstantMatrixType in {
}
let Class = CooperativeMatrixType in {
+ def : Property<"elementType", QualType> {
+ let Read = [{ node->getElementType() }];
+ }
def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index e6f396245070c..28e4b94c3bea9 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,7 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
-def CooperativeMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<Type>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 985cc3712c581..ddbf6cf17ac8f 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -213,6 +213,7 @@ def CIR_CastKind : CIR_I32Enum<"CastKind", "cast kind", [
I32EnumCase<"ptr_to_bool", 23>,
// CK_ToVoid
// CK_MatrixCast
+ // CK_CoopMatrixCast
// CK_VectorSplat
I32EnumCase<"integral", 27>,
I32EnumCase<"int_to_bool", 28>,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index bbaeb6ae37240..6853a2fd2e869 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8025,6 +8025,14 @@ class Sema final : public SemaBase {
QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
SourceLocation Loc, bool IsCompAssign);
+ /// Type checking for cooperative matrix binary operators.
+ QualType CheckCoopMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+ QualType CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign);
+
/// Are the two types SVE-bitcast-compatible types? I.e. is bitcasting from
/// the first SVE type (e.g. an SVE VLAT) to the second type (e.g. an SVE
/// VLST) allowed?
@@ -8037,6 +8045,11 @@ class Sema final : public SemaBase {
/// do they have the same number of rows and the same number of columns?
bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ /// Are the two types cooperative matrix types and do they have the same
+ /// dimensions i.e. do they have the same number of rows and the same number
+ /// of columns? Also do they have the same scope and use?
+ bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+
bool areVectorTypesSameSize(QualType srcType, QualType destType);
/// Are the two types lax-compatible vector types? That is, given
@@ -8079,6 +8092,13 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
+ // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
+ // We allow casting between cooperative matrixes of the same scope, use, and
+ // same dimensions i.e. when they have the same number of rows and columns.
+ // Returns true if the cast is invalid.
+ bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind);
+
// CheckVectorCast - check type constraints for vectors.
// Since vectors are an extension, there are no C standard reference for this.
// We allow casting between vectors and integer datatypes of the same size.
@@ -15397,8 +15417,11 @@ class Sema final : public SemaBase {
SourceLocation AttrLoc);
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc, Expr *Scope = nullptr,
- Expr *Use = nullptr, bool IsCoopMat = false);
+ SourceLocation AttrLoc);
+
+ QualType BuildCoopMatrixType(QualType T, Expr *Scope, Expr *Use,
+ Expr *NumRows, Expr *NumColumns,
+ SourceLocation AttrLoc);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index a7e096a4339cf..0317ce1268150 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2218,9 +2218,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
case Type::CooperativeMatrix: {
const auto *MT = cast<CooperativeMatrixType>(T);
TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
- // The internal layout of a matrix value is implementation defined.
- // Initially be ABI compatible with arrays with respect to alignment and
- // size.
+ // The internal layout of a cooperative matrix value is implementation
+ // defined. Initially be ABI compatible with arrays with respect to
+ // alignment and size.
Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
Align = ElementInfo.Align;
break;
@@ -10815,10 +10815,10 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatMatrixTypes - Return true if the two specified matrix types are
-/// compatible.
-static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
- const CooperativeMatrixType *RHS) {
+/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
+/// matrix types are compatible.
+static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
return LHS->getElementType() == RHS->getElementType() &&
LHS->getScope() == RHS->getScope() &&
@@ -12342,8 +12342,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
return LHS;
return {};
case Type::CooperativeMatrix:
- if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
- RHSCan->castAs<CooperativeMatrixType>()))
+ if (areCompatCoopMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
return LHS;
return {};
case Type::ObjCObject: {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 6b9aef5d479a6..db07fa0be8387 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1950,6 +1950,7 @@ bool CastExpr::CastConsistency() const {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
assert(!getType()->isBooleanType() && "unheralded conversion to bool");
goto CheckNoBasePath;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index dfd84f62ada25..45125f020612a 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -20212,6 +20212,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointCast:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLAggregateSplatCast:
llvm_unreachable("invalid cast kind for integral value");
@@ -21144,6 +21145,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLElementwiseCast:
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 020564f3edfb7..1c4a648003846 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -510,20 +510,6 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns), Layout(Layout) {}
-CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
- nColumns, use, canonType) {}
-
-CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
- unsigned scope, unsigned nRows,
- unsigned nColumns, unsigned use,
- QualType canonType)
- : MatrixType(tc, matrixType, canonType), NumRows(nRows),
- NumColumns(nColumns), Scope(scope), Use(use) {}
-
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -3304,8 +3290,6 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
- if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
- return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 51533f9dc0f9c..6869facf0d95c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -259,6 +259,7 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
case CK_IntegralToFloating:
case CK_LValueBitCast:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_MemberPointerToBoolean:
case CK_NonAtomicToAtomic:
case CK_ObjCObjectLValueCast:
@@ -1688,6 +1689,7 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 45c1bf6564514..ecd89bd18a0db 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7695,58 +7695,68 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy) {
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+llvm::Value *
+CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy) {
+ assert(!Args.empty() &&
+ "Expected at least one argument for cooperative matrix builtin");
+
+ llvm::LLVMContext &Context = CGM.getLLVMContext();
+ llvm::Module &Module = CGM.getModule();
+
+ llvm::Type *ResultLLVMType = ConvertType(ResultTy);
+
+ SmallVector<llvm::Type *, 8> ArgTypes;
+ ArgTypes.reserve(Args.size());
+
+ for (llvm::Value *Arg : Args) {
+ assert(Arg && "Cooperative matrix builtin argument must not be null");
+ ArgTypes.push_back(Arg->getType());
+ }
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+ // Although the frontend builtin is declared as void(...), the LLVM-level
+ // builtin returns the cooperative matrix result directly.
+ llvm::FunctionType *BuiltinType =
+ llvm::FunctionType::get(ResultLLVMType, ArgTypes,
+ /*IsVarArg=*/false);
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
- "coopmat.broadcast");
+ llvm::FunctionCallee Builtin =
+ Module.getOrInsertFunction(BuiltinName, BuiltinType);
+
+ return Builder.CreateCall(Builtin, Args, "coopmat.result");
}
llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
llvm::Value *LHS,
llvm::Value *RHS,
QualType ResultTy) {
- auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
-
switch (Opcode) {
case BO_Add:
- return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_add", {LHS, RHS}, ResultTy);
case BO_Sub:
- return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub(LHS, RHS, "coopmat.isub");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_sub", {LHS, RHS}, ResultTy);
case BO_Mul:
- // mat * mat → element-wise FMul/IMul
- // mat * scalar → handled separately (see scalar path below)
+ // Matrix * scalar.
if (!RHS->getType()->isVectorTy()) {
- // mat * scalar: broadcast scalar then element-wise mul
- llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
- return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_mul", {LHS, RHS},
+ ResultTy);
}
- return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul(LHS, RHS, "coopmat.imul");
+
+ // Matrix * matrix.
+ return EmitCoopMatBuiltinCall("coop_mat_binary_mul", {LHS, RHS}, ResultTy);
case BO_Div:
- if (IsFloat)
- return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned)
- return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
- return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+ return EmitCoopMatBuiltinCall("coop_mat_binary_div", {LHS, RHS}, ResultTy);
default:
- llvm_unreachable("Unsupported cooperative matrix binary op");
+ llvm_unreachable("Unsupported cooperative matrix binary operation");
}
-}
\ No newline at end of file
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatNeg(llvm::Value *Operand,
+ QualType ResultTy) {
+ return EmitCoopMatBuiltinCall("coop_mat_scalar_neg", {Operand}, ResultTy);
+}
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index ff0ed1473d7df..734a623e9e0bf 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1092,6 +1092,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
case CK_BuiltinFnToFnPtr:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_IntToOCLSampler:
@@ -1548,6 +1549,7 @@ static bool castPreservesZero(const CastExpr *CE) {
// Language extensions.
case CK_VectorSplat:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_NonAtomicToAtomic:
case CK_AtomicToNonAtomic:
case CK_HLSLVectorTruncation:
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 30d693fb371cb..87877bed010c1 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -618,6 +618,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index e2ec07bbae711..5aab1dfd6ac0c 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1388,6 +1388,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 354143adf5c02..82d814e6b0795 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3015,7 +3015,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
CGF.EmitIgnoredExpr(E);
return nullptr;
}
- case CK_MatrixCast: {
+ case CK_MatrixCast:
+ case CK_CoopMatrixCast: {
return EmitScalarConversion(Visit(E), E->getType(), DestTy,
CE->getExprLoc());
}
@@ -3689,15 +3690,10 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
- if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
- llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy =
- E->getType()->getAs<CooperativeMatrixType>()->getElementType();
- if (CompTy->isFloatingType())
- return Builder.CreateFNeg(Val, "coopmat.fneg");
- else
- return Builder.CreateNeg(Val, "coopmat.ineg");
- }
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType())
+ return CGF.EmitCoopMatNeg(CGF.EmitScalarExpr(E->getSubExpr()),
+ E->getType());
+
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 7b69f7ff45010..50778d662d430 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5191,15 +5191,6 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
- /// Helper function for EmitCoopMatBinaryOp
- llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
- QualType CoopMatQTy);
-
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
- llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
- llvm::Value *RHS, QualType ResultTy);
-
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
@@ -5543,6 +5534,20 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
SourceLocation loc);
+ /// Helper function to emit coop matrix builtin call.
+ llvm::Value *EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
+ llvm::ArrayRef<llvm::Value *> Args,
+ QualType ResultTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
+ /// EmitCoopMatNeg - Emit the computation of the negate op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
+
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
/// point operation, expressed as the maximum relative error in ulp.
void SetFPAccuracy(llvm::Value *Val, float Accuracy);
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index c0ba358207641..540dbc069c62b 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -332,11 +332,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
+ assert(isa<ConstantMatrixType>(Ty) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
+ if (const auto *CMTy = dyn_cast<CooperativeMatrixType>(Ty))
+ return getTypeInfo(CMTy->getElementType());
+
// Enum types are distinct types. In C++ they have "underlying types",
// however they aren't related for TBAA.
if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 5850604f88207..6909370d7f109 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -137,10 +137,8 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
case Type::CooperativeMatrix: {
- const auto *MT = cast<CooperativeMatrixType>(QT);
- return Builder.getArrayType(convertType(MT->getElementType()),
- MT->getNumRows() * MT->getNumColumns(),
- ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ llvm::reportFatalInternalError(
+ "Cooperative Matrix type not supported in ABI lowering library");
}
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 73d1753349fd4..f6bb43482c705 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -48,7 +48,6 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
- const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index e8d4660fd36b2..ecc7805ddd446 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1079,6 +1079,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
case CK_ZeroToOCLOpaqueType:
case CK_IntToOCLSampler:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
return false;
case CK_BooleanToSignedIntegral:
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index c797bdb11e0cb..c472cbeac9ef9 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -1607,6 +1607,15 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
return TC_Success;
}
+ if (SrcType->isCooperativeMatrixType() &&
+ DestType->isCooperativeMatrixType()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind)) {
+ SrcExpr = ExprError();
+ return TC_Failed;
+ }
+ return TC_Success;
+ }
+
if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
DestType == Self.Context.getLogicalOperationType()) {
SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBuiltIn(SrcExpr.get());
@@ -3247,6 +3256,13 @@ void CastOperation::CheckCStyleCast() {
return;
}
+ if (DestType->getAs<CooperativeMatrixType>() ||
+ SrcType->getAs<CooperativeMatrixType>()) {
+ if (Self.CheckCoopMatrixCast(OpRange, DestType, SrcType, Kind))
+ SrcExpr = ExprError();
+ return;
+ }
+
if (const VectorType *DestVecTy = DestType->getAs<VectorType>()) {
if (Self.CheckAltivecInitFromScalar(OpRange, DestType, SrcType)) {
SrcExpr = ExprError();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 85b5e8aba592f..cfabb76a2793a 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17458,7 +17458,7 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
ArgError = true;
} else {
ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
- if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
ArgError = true;
}
}
@@ -17578,7 +17578,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
M2Ty->getElementType().getUnqualifiedType())
Diag(Loc, diag::err_coop_matrix_element_type);
- if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
}
@@ -17593,7 +17593,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17715,9 +17715,8 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
- auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
- if (!ConstMType && !CoopMType) {
+ auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
+ if (!MType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17726,23 +17725,12 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- if (ConstMType) {
- QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
- if (CoopMType) {
- QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(), CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
-
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
- }
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 343d260fb1a26..67ef14deefed5 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14643,7 +14643,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
- if (!VDecl->getType()->isMatrixType()) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 33e9f21101b77..71f11b7103492 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5186,7 +5186,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
}
// If the base is a matrix type, try to create a new MatrixSubscriptExpr.
- if (base->getType()->isMatrixType()) {
+ if (base->getType()->isMatrixType() ||
+ base->getType()->isCooperativeMatrixType()) {
if (CheckAndReportCommaError(ArgExprs.front()))
return ExprError();
@@ -8060,16 +8061,6 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
- if (srcTy->isCooperativeMatrixType()) {
- const CooperativeMatrixType *matSrcType =
- srcTy->getAs<CooperativeMatrixType>();
- const CooperativeMatrixType *matDestType =
- destTy->getAs<CooperativeMatrixType>();
-
- return matSrcType->getNumRows() == matDestType->getNumRows() &&
- matSrcType->getNumColumns() == matDestType->getNumColumns();
- }
-
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -8077,6 +8068,22 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
+bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
+ QualType destTy) {
+ if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
+ return false;
+
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getScope() == matDestType->getScope() &&
+ matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns() &&
+ matSrcType->getUse() == matDestType->getUse();
+}
+
bool Sema::areVectorTypesSameSize(QualType SrcTy, QualType DestTy) {
assert(DestTy->isVectorType() || SrcTy->isVectorType());
@@ -8182,6 +8189,27 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
return false;
}
+bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
+ CastKind &Kind) {
+ if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
+ if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
+ << DestTy << SrcTy << R;
+ }
+ } else if (SrcTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << SrcTy << DestTy << R;
+ } else if (DestTy->isCooperativeMatrixType()) {
+ return Diag(R.getBegin(),
+ diag::err_invalid_conversion_between_matrix_and_type)
+ << DestTy << SrcTy << R;
+ }
+
+ Kind = CK_CoopMatrixCast;
+ return false;
+}
+
bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
CastKind &Kind) {
assert(VectorTy->isVectorType() && "Not a vector type!");
@@ -11410,6 +11438,9 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
ArithConvKind::Arithmetic);
if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
+ if (!IsDiv &&
+ (LHSTy->isCooperativeMatrixType() || RHSTy->isCooperativeMatrixType()))
+ return CheckCoopMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
@@ -11818,7 +11849,7 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -11973,7 +12004,7 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
if (LHS.get()->getType()->isCooperativeMatrixType() ||
RHS.get()->getType()->isCooperativeMatrixType()) {
QualType compType =
- CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
if (CompLHSTy)
*CompLHSTy = compType;
return compType;
@@ -14026,9 +14057,10 @@ QualType Sema::CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS,
return InvalidOperands(Loc, LHS, RHS);
}
-QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
- SourceLocation Loc,
- bool IsCompAssign) {
+QualType Sema::CheckCoopMatrixElementwiseOperands(ExprResult &LHS,
+ ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
if (!IsCompAssign) {
LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
if (LHS.isInvalid())
@@ -14038,33 +14070,55 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
- if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
- auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
- auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) &&
- "At least one operand must be a matrix");
- if (LHSMatType && RHSMatType) {
- if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
- return InvalidOperands(Loc, LHS, RHS);
+ // For conversion purposes, we ignore any qualifiers.
+ // For example, "const float" and "float" are equivalent.
+ QualType LHSType = LHS.get()->getType().getUnqualifiedType();
+ QualType RHSType = RHS.get()->getType().getUnqualifiedType();
- if (Context.hasSameType(LHSMatType, RHSMatType))
- return Context.getCommonSugaredType(
- LHS.get()->getType().getUnqualifiedType(),
- RHS.get()->getType().getUnqualifiedType());
+ const CooperativeMatrixType *LHSMatType =
+ LHSType->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *RHSMatType =
+ RHSType->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a cooperative matrix");
- QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
- if (!Context.hasSameType(LHSELTy, RHSELTy))
- return InvalidOperands(Loc, LHS, RHS);
+ if (Context.hasSameType(LHSType, RHSType))
+ return Context.getCommonSugaredType(LHSType, RHSType);
- return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy),
- LHSMatType->getScope(), LHSMatType->getNumRows(),
- RHSMatType->getNumColumns(), LHSMatType->getUse());
- }
+ // Type conversion may change LHS/RHS. Keep copies to the original results, in
+ // case we have to return InvalidOperands.
+ ExprResult OriginalLHS = LHS;
+ ExprResult OriginalRHS = RHS;
+ if (LHSMatType && !RHSMatType) {
+ RHS = tryConvertExprToType(RHS.get(), LHSMatType->getElementType());
+ if (!RHS.isInvalid())
+ return LHSType;
+
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
}
+ if (!LHSMatType && RHSMatType) {
+ LHS = tryConvertExprToType(LHS.get(), RHSMatType->getElementType());
+ if (!LHS.isInvalid())
+ return RHSType;
+ return InvalidOperands(Loc, OriginalLHS, OriginalRHS);
+ }
+
+ return InvalidOperands(Loc, LHS, RHS);
+}
+
+QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -14090,6 +14144,43 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
}
+QualType Sema::CheckCoopMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
+ SourceLocation Loc,
+ bool IsCompAssign) {
+ if (!IsCompAssign) {
+ LHS = DefaultFunctionArrayLvalueConversion(LHS.get());
+ if (LHS.isInvalid())
+ return QualType();
+ }
+ RHS = DefaultFunctionArrayLvalueConversion(RHS.get());
+ if (RHS.isInvalid())
+ return QualType();
+
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ return CheckCoopMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+}
+
static bool isLegalBoolVectorBinaryOp(BinaryOperatorKind Opc) {
switch (Opc) {
default:
@@ -15868,7 +15959,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
// Cooperative matrix support
- if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ if (LHSTy->isCooperativeMatrixType() && RHSTy->isCooperativeMatrixType()) {
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
@@ -15876,7 +15967,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
@@ -15906,7 +15997,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
- if (!LHSExpr->getType()->isMatrixType()) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
@@ -16576,7 +16667,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
- if (Opc == UO_Minus && Ty->isMatrixType()) {
+ if (Opc == UO_Minus && Ty->isCooperativeMatrixType()) {
return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
{InputExpr});
}
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index f44bdf45d1c91..77284c34f4ea7 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2459,9 +2459,11 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
if (!MA)
return TemplateDeductionResult::NonDeducedMismatch;
- // Check that the dimensions are the same
- if (MP->getNumRows() != MA->getNumRows() ||
- MP->getNumColumns() != MA->getNumColumns()) {
+ // Check that the dimensions, scope and use are the same
+ if (MP->getScope() != MA->getScope() ||
+ MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns() ||
+ MP->getUse() != MA->getUse()) {
return TemplateDeductionResult::NonDeducedMismatch;
}
// Perform deduction on element types.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 476f4d24fc1ec..9e956ad0af304 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2527,11 +2527,9 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc, Expr *ScopeExpr,
- Expr *UseExpr, bool IsCoopMat) {
- if (!IsCoopMat)
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc) {
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2612,29 +2610,76 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
- if (IsCoopMat) {
- std::optional<llvm::APSInt> ValueScope =
- ScopeExpr->getIntegerConstantExpr(Context);
- unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
- std::optional<llvm::APSInt> ValueUse =
- UseExpr->getIntegerConstantExpr(Context);
- unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
-
- if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr)
- << ColRange << "matrix scope";
- return QualType();
- }
- if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
- return QualType();
- }
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
- }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
+QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
+ Expr *NumRows, Expr *NumCols, Expr *UseExpr,
+ SourceLocation AttrLoc) {
+ std::optional<llvm::APSInt> ValueRows =
+ NumRows->getIntegerConstantExpr(Context);
+ std::optional<llvm::APSInt> ValueColumns =
+ NumCols->getIntegerConstantExpr(Context);
+
+ auto const RowRange = NumRows->getSourceRange();
+ auto const ColRange = NumCols->getSourceRange();
+
+ // Both are row and column expressions are invalid.
+ if (!ValueRows && !ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange << ColRange;
+ return QualType();
+ }
+
+ // Only the row expression is invalid.
+ if (!ValueRows) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << RowRange;
+ return QualType();
+ }
+
+ // Only the column expression is invalid.
+ if (!ValueColumns) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant << ColRange;
+ return QualType();
+ }
+
+ // Check the matrix dimensions.
+ unsigned MatrixRows = static_cast<unsigned>(ValueRows->getZExtValue());
+ unsigned MatrixColumns = static_cast<unsigned>(ValueColumns->getZExtValue());
+ if (MatrixRows == 0 && MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size)
+ << "matrix" << RowRange << ColRange;
+ return QualType();
+ }
+ if (MatrixRows == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << RowRange;
+ return QualType();
+ }
+ if (MatrixColumns == 0) {
+ Diag(AttrLoc, diag::err_attribute_zero_size) << "coop_mat" << ColRange;
+ return QualType();
+ }
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+}
+
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
if ((T->isArrayType() && !getLangOpts().allowArrayReturnTypes()) ||
T->isFunctionType()) {
@@ -6009,7 +6054,7 @@ static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
}
}
- llvm_unreachable("no matrix_type attribute found at the expected location!");
+ llvm_unreachable("no coop_mat attribute found at the expected location!");
}
namespace {
@@ -9023,34 +9068,40 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S, bool IsCoopMat = false) {
- if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
+ Sema &S) {
+ if (!S.getLangOpts().MatrixTypes) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- unsigned int NumAttrs = IsCoopMat ? 4 : 2;
- if (Attr.getNumArgs() != NumAttrs) {
+ if (Attr.getNumArgs() != 2) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << NumAttrs;
+ << Attr << 2;
return;
}
- if (IsCoopMat) {
- Expr *Scope = Attr.getArgAsExpr(0);
- Expr *RowsExpr = Attr.getArgAsExpr(1);
- Expr *ColsExpr = Attr.getArgAsExpr(2);
- Expr *Use = Attr.getArgAsExpr(3);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
- Scope, Use, /* IsCoopMat */ true);
- if (!T.isNull())
- CurType = T;
- } else {
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
+}
+
+/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
+ Sema &S) {
+ if (Attr.getNumArgs() != 4) {
+ S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+ << Attr << 4;
+ return;
}
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildCoopMatrixType(CurType, Scope, RowsExpr, ColsExpr, Use,
+ Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9335,7 +9386,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
break;
case ParsedAttr::AT_CoopMatrixType:
- HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ HandleCoopMatrixTypeAttr(type, attr, state.getSema());
attr.setUsedAsTypeAttr();
break;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 43adeebd94500..f4c78b965d24a 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -30,7 +30,7 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
// ---------------------------------------------------------------------------
-// 1. Basic type construction — four use roles, float element type.
+// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
@@ -43,7 +43,7 @@ typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
// AST: TypedefDecl {{.*}} MatA_int8x8
// ---------------------------------------------------------------------------
-// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// Type printer — VarDecls carry the coop_mat attribute in their type string.
// ---------------------------------------------------------------------------
void test_type_spelling(void) {
MatA_float16x16 a;
@@ -58,17 +58,7 @@ void test_type_spelling(void) {
// AST: VarDecl {{.*}} d {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
-// float(4B)*16*16 = 1024 B = 8192 bits
-// int(4B)*8*8 = 256 B = 2048 bits
-// ---------------------------------------------------------------------------
-void test_sizeof(void) {
- _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
- _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
-}
-
-// ---------------------------------------------------------------------------
-// 4. Parameter / return type — type preserved through function boundary.
+// Parameter / return type — type preserved through function boundary.
// ---------------------------------------------------------------------------
MatA_float16x16 test_param_return(MatA_float16x16 in) {
return in;
@@ -78,7 +68,7 @@ MatA_float16x16 test_param_return(MatA_float16x16 in) {
// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 5. TypeLoc operand traversal — all four operand slots populated.
+// TypeLoc operand traversal — all four operand slots populated.
// ---------------------------------------------------------------------------
void test_typeloc_operands(void) {
float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
@@ -88,7 +78,7 @@ void test_typeloc_operands(void) {
// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// RecursiveASTVisitor — element type (half) reachable through the node.
// ---------------------------------------------------------------------------
void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
(void)x;
@@ -97,8 +87,8 @@ void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))
// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
// ---------------------------------------------------------------------------
-// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
-// same canonical type; taking a pointer across them compiles cleanly.
+// mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
@@ -109,6 +99,6 @@ void test_merge_types(void) {
}
// ---------------------------------------------------------------------------
-// 8. PCH serialisation round-trip.
+// PCH serialisation round-trip.
// ---------------------------------------------------------------------------
// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 951fedea44ee4..dac8117f971d2 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1912,7 +1912,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
-DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, Type)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
>From 125a9060506bbf0bd498ae2e1397d9075304b7f4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 14:31:27 -0400
Subject: [PATCH 15/43] iMinor issues with previous commit
---
clang/include/clang/AST/TypeLoc.h | 2 +-
clang/lib/CodeGen/CGExpr.cpp | 2 +-
clang/lib/Sema/SemaChecking.cpp | 3 +--
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 122090f7e6ec5..b7432d03a92dd 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2217,7 +2217,7 @@ class CooperativeMatrixTypeLoc
public:
/// The location of the attribute name, i.e.
/// float __attribute__((coop_mat(0, 4, 2, 1)))
- /// ^^~~~~~~~~~~~~~~~~~~
+ /// ^~~~~~~~~~~~~~~~~~~~
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index ecd89bd18a0db..8f4f8317fd797 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -6417,6 +6417,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
@@ -7702,7 +7703,6 @@ CodeGenFunction::EmitCoopMatBuiltinCall(llvm::StringRef BuiltinName,
assert(!Args.empty() &&
"Expected at least one argument for cooperative matrix builtin");
- llvm::LLVMContext &Context = CGM.getLLVMContext();
llvm::Module &Module = CGM.getModule();
llvm::Type *ResultLLVMType = ConvertType(ResultTy);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index cfabb76a2793a..0fed3d8253a8d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17726,8 +17726,7 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
>From 2f0fe37a9eb9149382defd2c1d199efd2e78a229 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 15:35:07 -0400
Subject: [PATCH 16/43] Add regression test for change in SPIRVUtils.cpp
---
.../SPIRV/coop-matrix-builtin-demangle.ll | 51 +++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
diff --git a/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
new file mode 100644
index 0000000000000..e99d2199b7131
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/coop-matrix-builtin-demangle.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=spirv64-unknown-unknown \
+; RUN: -spirv-ext=+SPV_KHR_cooperative_matrix \
+; RUN: -filetype=asm < %s | FileCheck %s
+
+target triple = "spirv64-unknown-unknown"
+
+%coopmat.f32.sc3.16x16.u0 =
+ type target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+
+; ---------------------------------------------------------------------------
+; Suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_mangled_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_mangled_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
+; ---------------------------------------------------------------------------
+; No-suffix branch.
+; ---------------------------------------------------------------------------
+
+declare spir_func %coopmat.f32.sc3.16x16.u0
+ at __spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1), i32)
+
+define spir_kernel void @test_canonical_load(
+ ptr addrspace(1) %ptr) {
+entry:
+ %mat = call spir_func %coopmat.f32.sc3.16x16.u0
+ @__spirv_CooperativeMatrixLoadKHR(
+ ptr addrspace(1) %ptr, i32 16)
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function test_canonical_load
+; CHECK: OpCooperativeMatrixLoadKHR
+; CHECK: OpFunctionEnd
+
>From f1c24cdf1e82cfd2b4ea784a9eb50c25522d2e58 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 8 Sep 2026 17:00:22 -0400
Subject: [PATCH 17/43] Address review comments
---
clang/include/clang/AST/TypeBase.h | 19 ++------
clang/include/clang/AST/TypeProperties.td | 6 +--
.../include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/lib/AST/ASTStructuralEquivalence.cpp | 2 +-
clang/lib/Headers/opencl-c-base.h | 47 ++++++++++---------
5 files changed, 33 insertions(+), 43 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 0b83ac3fd03e4..5328e2db64d2c 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4603,13 +4603,8 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
- : CooperativeMatrixType(Type::CooperativeMatrix, ElementType, Scope,
- NumRows, NumColumns, Use, CanonicalType) {}
-
- CooperativeMatrixType(TypeClass TypeClass, QualType ElementType,
- unsigned Scope, unsigned NumRows, unsigned NumColumns,
- unsigned Use, QualType CanonicalType)
- : Type(TypeClass, CanonicalType, ElementType->getDependence()),
+ : Type(Type::CooperativeMatrix, CanonicalType,
+ ElementType->getDependence()),
ElementType(ElementType), NumRows(NumRows), NumColumns(NumColumns),
Scope(Scope), Use(Use) {}
@@ -4629,12 +4624,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns the number of elements required to embed the matrix into
- /// a vector representation.
- unsigned getNumElementsFlattened() const {
- return getNumRows() * getNumColumns();
- }
-
/// Returns true if \p NumElements is a valid cooperative matrix dimension.
static constexpr bool isDimensionValid(size_t NumElements) {
return NumElements > 0 && NumElements <= MaxElementsPerDimension;
@@ -4658,12 +4647,12 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
}
void Profile(llvm::FoldingSetNodeID &ID) {
- Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
}
static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
- unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Scope, unsigned NumRows, unsigned NumColumns,
unsigned Use, TypeClass TypeClass) {
ID.AddPointer(ElementType.getAsOpaquePtr());
ID.AddInteger(NumRows);
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index d134edc39e92c..1d486ba3c2058 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -267,15 +267,15 @@ let Class = CooperativeMatrixType in {
def : Property<"elementType", QualType> {
let Read = [{ node->getElementType() }];
}
- def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"scope", UInt32> { let Read = [{ node->getScope() }]; }
def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
def : Property<"numColumns", UInt32> {
let Read = [{ node->getNumColumns() }];
}
- def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+ def : Property<"use", UInt32> { let Read = [{ node->getUse() }]; }
def : Creator<[{
- return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ return ctx.getCooperativeMatrixType(elementType, scope, numRows, numColumns, use);
}]>;
}
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 8a583ad504c70..56ce32cf3e7c5 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 8ec1100f9f4d8..fd4f55e43f665 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1155,7 +1155,7 @@ bool ASTStructuralEquivalence::isEquivalent(
const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
// The element types must be structurally equivalent and the number of rows
- // and columns must match.
+ // and columns and scope and use must match.
if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
Mat2->getElementType()) ||
Mat1->getScope() != Mat2->getScope() ||
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 7b3d239299c6e..83bb35f1d886a 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,6 +632,30 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
+
#ifdef cl_intel_device_side_avc_motion_estimation
#define CLK_AVC_ME_MAJOR_16x16_INTEL 0x0
@@ -771,27 +795,4 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
-typedef enum coop_matrix_operands_t {
- CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
- CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
- CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
-} coop_matrix_operands_t;
-
-typedef enum coop_matrix_use_t {
- CLK_COOPERATIVE_MATRIX_A = 0,
- CLK_COOPERATIVE_MATRIX_B = 1,
- CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
-} coop_matrix_use_t;
-
-typedef enum coop_matrix_layout_t {
- CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
- CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
-} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From d9e7ccc3940ada259dab7b26bf378697f626e87d Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sat, 12 Sep 2026 17:21:24 -0400
Subject: [PATCH 18/43] Add cooperative matrix type support in missing sites
---
clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp | 1 +
clang/lib/CIR/CodeGen/CIRGenFunction.cpp | 2 ++
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 2 ++
4 files changed, 6 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 91535c57132bd..845e8c9810dd3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -536,6 +536,7 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
case CK_FixedPointToIntegral:
case CK_IntegralToFixedPoint:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
case CK_HLSLArrayRValue:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index cfe575f4670dd..94a17748e88a1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -648,6 +648,7 @@ class ConstExprEmitter
case CK_IntegralToFixedPoint:
case CK_ZeroToOCLOpaqueType:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_HLSLArrayRValue:
case CK_HLSLVectorTruncation:
case CK_HLSLMatrixTruncation:
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index e5e83a2647018..c7f97b344ccd8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -82,6 +82,7 @@ cir::TypeEvaluationKind CIRGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -1681,6 +1682,7 @@ void CIRGenFunction::emitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index 763ed026f8714..ae154e00752dd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -902,6 +902,7 @@ const char *vTableClassNameForType(const CIRGenModule &cgm, const Type *ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -1375,6 +1376,7 @@ mlir::Attribute CIRGenItaniumRTTIBuilder::buildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
>From 3c41a0a239aead9676ac1c91dc671660d29d853b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 21:44:28 -0400
Subject: [PATCH 19/43] Add cooperative matrix related changes to CIR
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 14 ++++++++++++++
clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 9 +++++++++
2 files changed, 23 insertions(+)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 97ce3fbcabcbc..f7202caf8e5f3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2330,6 +2330,20 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BI__builtin_masked_store:
case Builtin::BI__builtin_masked_compress_store:
case Builtin::BI__builtin_masked_scatter:
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
+ case Builtin::BIcoop_mat_length:
+ case Builtin::BI__builtin_matrix_transpose:
+ case Builtin::BI__builtin_matrix_column_major_load:
+ case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 836f27cf5345e..f65e94ffa88c1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -872,6 +872,10 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
}
mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
+ if (e->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ cgf.cgm.errorNYI(e->getSourceRange(), "cooperative matrix unary minus");
+ return {};
+ }
ignoreResultAssign = false;
mlir::Value operand;
if (!promotionType.isNull())
@@ -1236,6 +1240,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
mlir::Value VisitBin##OP(const BinaryOperator *e) { \
+ if (e->getType()->isCooperativeMatrixType()) { \
+ cgf.cgm.errorNYI(e->getSourceRange(), \
+ "cooperative matrix binary operator"); \
+ return {}; \
+ } \
QualType promotionTy = getPromotionType(e->getType()); \
auto result = emit##OP(emitBinOps(e, promotionTy)); \
if (result && !promotionTy.isNull()) \
>From 613f6c883501727e57e3992443cd2b3d40263203 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 17 Sep 2026 23:45:47 -0400
Subject: [PATCH 20/43] Address review comments: 1. Update comments to remove
function names 2. Add change to emit error if extension is not enabled using
'pragma OPENCL_EXTENSION ... enable' in OpenCL source
---
.../include/clang/Basic/DiagnosticSemaKinds.td | 3 +++
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
clang/include/clang/Sema/Sema.h | 9 ++++-----
clang/lib/AST/ASTContext.cpp | 3 +--
clang/lib/CodeGen/CodeGenFunction.h | 6 ++----
clang/lib/Sema/SemaType.cpp | 18 +++++++++++++++---
.../CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 ++
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 ++
.../coop-mat-init-length-codegen.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 ++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 +++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 ++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 ++
14 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 300fda4bed280..c80d5f2222a5f 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11989,6 +11989,9 @@ def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
// OpenCL C Cooperative Matrix.
+def ext_coopmat_without_pragma
+ : Error<"cooperative matrix types require OpenCL extension '%0' to be enabled via pragma">;
+
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 56ce32cf3e7c5..8a583ad504c70 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_khr_cooperative_matrix, false, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6853a2fd2e869..6da59918d8854 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8092,10 +8092,10 @@ class Sema final : public SemaBase {
bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
- // CheckCoopMatrixCast - Check type constraints for cooperative matrix casts.
- // We allow casting between cooperative matrixes of the same scope, use, and
- // same dimensions i.e. when they have the same number of rows and columns.
- // Returns true if the cast is invalid.
+ /// Check type constraints for cooperative matrix casts. We allow casting
+ /// between cooperative matrixes of the same scope, use, and same dimensions
+ /// i.e. when they have the same number of rows and columns. Returns true if
+ /// the cast is invalid.
bool CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind);
@@ -15415,7 +15415,6 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
-
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
SourceLocation AttrLoc);
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0317ce1268150..c1086d2ed5e4e 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10815,8 +10815,7 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
-/// areCompatCoopMatrixTypes - Return true if the two specified cooperative
-/// matrix types are compatible.
+/// Return true if the two specified cooperative matrix types are compatible.
static bool areCompatCoopMatrixTypes(const CooperativeMatrixType *LHS,
const CooperativeMatrixType *RHS) {
assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 50778d662d430..e689a751b7a9c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5539,13 +5539,11 @@ class CodeGenFunction : public CodeGenTypeCache {
llvm::ArrayRef<llvm::Value *> Args,
QualType ResultTy);
- /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
- /// returning the result.
+ /// Emit the computation of the specified binary op, returning the result.
llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
llvm::Value *RHS, QualType ResultTy);
- /// EmitCoopMatNeg - Emit the computation of the negate op,
- /// returning the result.
+ /// Emit the computation of the negate op, returning the result.
llvm::Value *EmitCoopMatNeg(llvm::Value *Operand, QualType ResultTy);
/// SetFPAccuracy - Set the minimum required accuracy of the given floating
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 9e956ad0af304..2f2b16c122b44 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2676,8 +2676,19 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
return QualType();
}
- return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
- MatrixColumns, Use);
+ auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ if (getLangOpts().OpenCL) {
+ if (!getOpenCLOptions().isSupported("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::err_opencl_requires_extension)
+ << 0 << Result << "cl_khr_cooperative_matrix";
+ else if (!getOpenCLOptions().isAvailableOption("cl_khr_cooperative_matrix",
+ getLangOpts()))
+ Diag(AttrLoc, diag::ext_coopmat_without_pragma)
+ << "cl_khr_cooperative_matrix";
+ }
+ return Result;
}
bool Sema::CheckFunctionReturnType(QualType T, SourceLocation Loc) {
@@ -9079,6 +9090,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
<< Attr << 2;
return;
}
+
Expr *RowsExpr = Attr.getArgAsExpr(0);
Expr *ColsExpr = Attr.getArgAsExpr(1);
QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
@@ -9086,7 +9098,7 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
CurType = T;
}
-/// HandleCoopMatrixTypeAttr - "coop_mat" attribute, like ext_vector_type
+/// "coop_mat" attribute, like ext_vector_type
static void HandleCoopMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Sema &S) {
if (Attr.getNumArgs() != 4) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9d55bd8c61cc3..c070010c24b49 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -19,6 +19,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Matrix type aliases used across tests
// ---------------------------------------------------------------------------
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index ad8df2939269f..3dada9b4ef5de 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -14,6 +14,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index dfa763b060e72..0b7a825b5d369 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -5,6 +5,8 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index e8fad815806ec..38d98b28482bf 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -33,6 +33,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index 799f7a9e15a35..b32f6549deba5 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -15,6 +15,8 @@
// ---------------------------------------------------------------------------
// Type definitions shared across all tests.
// ---------------------------------------------------------------------------
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 0c27978d492a9..351728d245494 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -12,6 +12,13 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+// ---------------------------------------------------------------------------
+// 0. Use of coop mat type without specifying pragma
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
+
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index bcedd94fdbf17..dafa5b3bf123a 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -16,6 +16,8 @@
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index f4c78b965d24a..3f8b221c3edeb 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -29,6 +29,8 @@
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
+
// ---------------------------------------------------------------------------
// Basic type construction — four use roles, float element type.
// ---------------------------------------------------------------------------
>From dd739ca3b5c1cc89e63e59de64a6fac1583de3d4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 00:21:21 -0400
Subject: [PATCH 21/43] Address review comments:
1. Fix Matrix Scalar multiply for cases where LHS is scalar
2. Use getBuiltinID() to check for coop mat builtins instead of name
---
clang/lib/Sema/SemaDecl.cpp | 16 ++++++++++++++--
clang/lib/Sema/SemaExpr.cpp | 4 +++-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 ++++
3 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 67ef14deefed5..0e05bdb8729c6 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14114,9 +14114,21 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
if (!Fname)
return false;
- if (Fname->getName().starts_with("coop_mat") &&
- !Fname->getName().starts_with("coop_mat_length"))
+ switch (F->getBuiltinID()) {
+ case Builtin::BIcoop_mat_load:
+ case Builtin::BIcoop_mat_store:
+ case Builtin::BIcoop_mat_mulAdd:
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ case Builtin::BIcoop_mat_scalar_mul:
+ case Builtin::BIcoop_mat_scalar_neg:
+ case Builtin::BIcoop_mat_init:
return true;
+ default:
+ return false;
+ }
return false;
}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 71f11b7103492..872df233400ba 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15967,8 +15967,10 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
- } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType())
+ } else if (LHSTy->isCooperativeMatrixType() && RHSTy->isScalarType()) {
return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (RHSTy->isCooperativeMatrixType() && LHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, RHSExpr, LHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 3dada9b4ef5de..28f7660957a62 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -79,9 +79,13 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
a = coop_mat_load(ptr, ROW_MAJOR, 16);
r = a * s;
(void)r;
+ MatA_t r2;
+ r2 = s * a;
+ (void)r2;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
>From caf55448a5fde647e7cdf8278d633103dd97de48 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 18 Sep 2026 02:24:09 -0400
Subject: [PATCH 22/43] Address more review comments: 1. Add documentation for
'coop_mat(...)' attribute 2. Add more checks for scope, use, and element type
3. Fix some typos
---
clang/include/clang/Basic/Attr.td | 2 +-
clang/include/clang/Basic/AttrDocs.td | 47 +++++++++++++++++++
.../clang/Basic/DiagnosticSemaKinds.td | 5 +-
clang/lib/Sema/SemaChecking.cpp | 15 +++---
clang/lib/Sema/SemaType.cpp | 22 ++++++++-
clang/test/AST/undocumented-attrs.cpp | 3 +-
6 files changed, 81 insertions(+), 13 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index f66fc29fdac45..14c0452bc0a15 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3986,7 +3986,7 @@ def CoopMatrixType : TypeAttr {
let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
- let Documentation = [Undocumented];
+ let Documentation = [CoopMatrixTypeDocs];
}
def Visibility : InheritableAttr {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 0ab1429ff09d6..93174cae1d539 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1479,6 +1479,53 @@ See the OpenCL documentation for some more complete examples.
}];
}
+def CoopMatrixTypeDocs : Documentation {
+ let Category = DocCatType;
+ let Heading = "coop_mat";
+ let Content = [{
+The `coop_mat` attribute declares an OpenCL cooperative matrix type. Cooperative
+matrix operations are distributed across a set of invocations, such as a
+subgroup, allowing the implementation to optimize matrix multiply operations.
+
+The attribute is applied to a scalar numerical component type and takes four
+arguments:
+
+```c
+__attribute__((coop_mat(scope, rows, columns, use)))
+```
+- scope specifies the memory scope of the cooperative matrix.
+ It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+- rows specifies the number of rows in the matrix. It must be a constant expression.
+- columns specifies the number of columns in the matrix. It must be a constant expression.
+- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
+ CLK_COOPERATIVE_MATRIX_A
+ CLK_COOPERATIVE_MATRIX_B
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+The coop_mat attribute can be used to declare local variables, function parameters, and function return values. Cooperative matrix types must not be used as kernel arguments.
+
+```c
+float vmat1 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 8,
+ 8,
+ CLK_COOPERATIVE_MATRIX_A)));
+
+typedef half chmat44 __attribute__((coop_mat(
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 4,
+ 4,
+ CLK_COOPERATIVE_MATRIX_B)));
+chmat44 vmat2;
+```
+The base component type must be a scalar numerical type.
+Implicit and explicit casts between cooperative matrix types are not generally permitted.
+Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+See the OpenCL documentation for some more complete examples.
+ }];
+}
+
def DiagnoseIfDocs : Documentation {
let Category = DocCatFunction;
let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c80d5f2222a5f..c696c0e6a9252 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12027,9 +12027,12 @@ def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
"CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
-def err_coop_matrix_element_type
+def err_mismatched_coop_matrix_element_type
: Error<"inconsistent cooperative matrix element type">;
+def err_invalid_coop_matrix_element_type
+ : Error<"invalid cooperative matrix element type">;
+
def err_coop_matrix_row_or_col_mismatch
: Error<"mismatch of cooperative matrix row or column">;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0fed3d8253a8d..0ce7084df1f89 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17541,7 +17541,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
Arg0->getBeginLoc());
@@ -17576,7 +17576,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (MOutTy->getElementType().getUnqualifiedType() !=
M2Ty->getElementType().getUnqualifiedType())
- Diag(Loc, diag::err_coop_matrix_element_type);
+ Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
@@ -17605,7 +17605,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType()) {
- Diag(ALoc, diag::err_coop_matrix_element_type);
+ Diag(ALoc, diag::err_mismatched_coop_matrix_element_type);
return true;
}
return false;
@@ -17619,8 +17619,9 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
- CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ if (CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(),
+ Arg1->getType(), Arg1->getBeginLoc()))
+ return ExprError();
TheCall->setType(Arg0->getType());
@@ -17668,11 +17669,11 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
- return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
- return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+ return ExprError(Diag(Loc1, diag::err_mismatched_coop_matrix_element_type));
if (M0Ty->getNumRows() != M2Ty->getNumRows())
return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 2f2b16c122b44..8bc9451365c06 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2663,17 +2663,35 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
}
std::optional<llvm::APSInt> ValueScope =
ScopeExpr->getIntegerConstantExpr(Context);
+ // Only the scope expression is invalid.
+ if (!ValueScope) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << ScopeExpr->getSourceRange();
+ return QualType();
+ }
unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
std::optional<llvm::APSInt> ValueUse =
UseExpr->getIntegerConstantExpr(Context);
+ // Only the use expression is invalid.
+ if (!ValueUse) {
+ Diag(AttrLoc, diag::err_attribute_argument_type)
+ << "coop_mat" << AANT_ArgumentIntegerConstant
+ << UseExpr->getSourceRange();
+ return QualType();
+ }
unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix scope";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Scope << "matrix scope";
return QualType();
}
if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Use << "matrix use";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
+ Diag(AttrLoc, diag::err_invalid_coop_matrix_element_type) << ElementTy;
return QualType();
}
auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index dff9d3845062a..ed82ff3f517fd 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -27,7 +27,6 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
-CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -89,4 +88,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 83
+CHECK-NEXT: Total: 82
\ No newline at end of file
>From fa13584ed49cca91dbbd99391e9a6325730279fe Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 12:24:29 -0400
Subject: [PATCH 23/43] Add missing checks for coop_mat builtins in
SemaChecking.cpp
1. Stride check in load/store
2. Check for optional matrix operand in muladd
---
.../clang/Basic/DiagnosticSemaKinds.td | 7 ++++++
clang/include/clang/Sema/Sema.h | 1 +
clang/lib/Sema/SemaChecking.cpp | 24 ++++++++++++++++++-
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c696c0e6a9252..6612665163b67 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12007,6 +12007,10 @@ def err_coop_mem_layout_enum
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
"CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+def err_coop_mat_stride_type
+ : Error<"stride argument for load/store of cooperative matrix must have "
+ "type 'size_t'">;
+
def err_coop_element_and_pointer_type
: Error<"inconsistent between cooperative matrix element type and buffer "
"pointer type">;
@@ -12014,6 +12018,9 @@ def err_coop_element_and_pointer_type
def err_coop_matrix_arg
: Error<"argument must be a valid cooperative matrix type">;
+def err_coop_matrix_operands_type
+ : Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
+
def err_coop_matrix_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6da59918d8854..7d00a598ea2e7 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3120,6 +3120,7 @@ class Sema final : public SemaBase {
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 0ce7084df1f89..4b39ef832298d 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17524,6 +17524,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
+bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+ if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
+ return true;
+ }
+ return false;
+}
+
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
@@ -17532,6 +17540,8 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ return ExprError();
return CallResult;
}
@@ -17547,6 +17557,8 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
+ if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ return ExprError();
return CallResult;
}
@@ -17638,9 +17650,19 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
ExprResult CallResult) {
- if (checkArgCount(TheCall, 3))
+ if (checkArgCountRange(TheCall, 3, 4))
return ExprError();
+ // The fourth argument is optional. If present, it must be an enum.
+ if (TheCall->getNumArgs() == 4) {
+ Expr *Operands = TheCall->getArg(3);
+ QualType OperandsTy = Operands->getType().getCanonicalType();
+
+ if (!OperandsTy->isEnumeralType()) {
+ Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ }
+ }
+
Expr *Arg0 = TheCall->getArg(0);
Expr *Arg1 = TheCall->getArg(1);
Expr *Arg2 = TheCall->getArg(2);
>From 09de00f2c664559911c32d612e9eb35777827cb4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 21 Sep 2026 13:45:20 -0400
Subject: [PATCH 24/43] Merge type lowering for cooperative matrix between
CGBuiltin.cpp and CodeGenTypes.cpp
---
clang/lib/CodeGen/CGBuiltin.cpp | 49 +++++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.cpp | 46 +++++++++++-----------------
clang/lib/CodeGen/CodeGenTypes.h | 5 +++
3 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index b6d79bd240f06..c3c18e616218b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2789,20 +2789,13 @@ static std::string getSPIRVBuiltinName(unsigned BuiltinID,
return "";
}
-static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
- CodeGenModule &CGM,
+static llvm::TargetExtType *getTargetExtType(CodeGenModule &CGM,
const CooperativeMatrixType *MTy) {
- llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
- // Type arguments for TargetExtType
- llvm::Type *Tys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = llvm::TargetExtType::get(
- CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
- return RetType;
+ llvm::Type *Ty = CGM.getTypes().ConvertTypeForCoopMat(MTy);
+ auto *TargetTy = llvm::dyn_cast<llvm::TargetExtType>(Ty);
+ if (!TargetTy)
+ return nullptr;
+ return TargetTy;
}
/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
@@ -4737,7 +4730,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(1));
auto Stride = EmitScalarExpr(E->getArg(2));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
@@ -4773,7 +4766,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Layout = EmitScalarExpr(E->getArg(2));
auto Stride = EmitScalarExpr(E->getArg(3));
// Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ llvm::TargetExtType *ArgType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
@@ -4817,10 +4810,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
- auto *CType = getTargetExtType(*this, CGM, MCTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
+ auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
@@ -4848,9 +4841,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
- auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
+ auto *BType = getTargetExtType(CGM, MBTy);
llvm::Type *ElTy = ConvertType(MTy->getElementType());
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4884,8 +4877,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg1 = EmitScalarExpr(E->getArg(1));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
auto *BType = Arg1->getType();
// Check if the data is signed/unsigned
@@ -4915,8 +4908,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = EmitScalarExpr(E->getArg(0));
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *RetType = getTargetExtType(*this, CGM, MTy);
- auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *RetType = getTargetExtType(CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MATy);
// Check if the data is signed/unsigned
QualType QT = MTy->getElementType();
@@ -4944,7 +4937,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_init: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
auto Init = EmitScalarExpr(E->getArg(0));
- auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *RetType = getTargetExtType(CGM, MTy);
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
@@ -4962,7 +4955,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_length: {
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
- auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 6ac24c6554922..c5e4017b268f8 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,6 +90,20 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
+llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+ llvm::Type *ElementTy = ConvertType(CMT->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElementTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {CMT->getScope(), CMT->getNumRows(), CMT->getNumColumns(),
+ CMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
@@ -125,21 +139,8 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
- if (T->isCooperativeMatrixType()) {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent a Coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
- return CoopMatType;
- }
+ if (const CooperativeMatrixType *CMT = T->getAs<CooperativeMatrixType>())
+ return ConvertTypeForCoopMat(CMT);
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -719,20 +720,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
break;
}
case Type::CooperativeMatrix: {
- const CooperativeMatrixType *DMT =
- cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
- llvm::Type *ElTy = ConvertType(DMT->getElementType());
- // Type argument for TargetExtType
- llvm::Type *ArgTys[] = {ElTy};
- // Unsigned arguments for TargetExtType
- unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
- DMT->getUse()};
- // Create a TargetExtType to represent the coop matrix type
- llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
- getLLVMContext(), "spirv.CooperativeMatrixKHR",
- llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
-
- ResultType = CoopMatType;
+ ResultType = ConvertTypeForCoopMat(T->getAs<CooperativeMatrixType>());
break;
}
case Type::FunctionNoProto:
diff --git a/clang/lib/CodeGen/CodeGenTypes.h b/clang/lib/CodeGen/CodeGenTypes.h
index 860f1f1ccdce2..dfe0df716bbf3 100644
--- a/clang/lib/CodeGen/CodeGenTypes.h
+++ b/clang/lib/CodeGen/CodeGenTypes.h
@@ -126,6 +126,11 @@ class CodeGenTypes {
/// ConvertType - Convert type T into a llvm::Type.
llvm::Type *ConvertType(QualType T);
+ /// Convert a Clang cooperative matrix type to the corresponding LLVM type.
+ /// The element type and cooperative matrix properties, including scope,
+ /// dimensions, and use, are preserved in the resulting LLVM type.
+ llvm::Type *ConvertTypeForCoopMat(const CooperativeMatrixType *CMT);
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
>From 35968f2b75979d094a6ed083a373b3385db154ed Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 22 Sep 2026 12:22:50 -0400
Subject: [PATCH 25/43] Fix clang format issue
---
clang/lib/CodeGen/CodeGenTypes.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c5e4017b268f8..2147f06763ecb 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -90,7 +90,8 @@ void CodeGenTypes::addRecordTypeName(const RecordDecl *RD,
Ty->setName(OS.str());
}
-llvm::Type *CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
+llvm::Type *
+CodeGenTypes::ConvertTypeForCoopMat(const CooperativeMatrixType *CMT) {
llvm::Type *ElementTy = ConvertType(CMT->getElementType());
// Type arguments for TargetExtType
llvm::Type *Tys[] = {ElementTy};
>From 8bee2443a47ba081b86ccb00e5b17052250a791f Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 16:56:39 -0400
Subject: [PATCH 26/43] Add check for disallowing cooperative matrix type for
kernel parameters and update tests
---
clang/lib/Sema/SemaDecl.cpp | 3 +++
clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 10 ++++++++--
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 4 +++-
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 7 ++++++-
5 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 0e05bdb8729c6..19620ebd70da4 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -9868,6 +9868,9 @@ static bool isOpenCLSizeDependentType(ASTContext &C, QualType Ty) {
}
static OpenCLParamType getOpenCLKernelParameterType(Sema &S, QualType PT) {
+ if (PT->isCooperativeMatrixType())
+ return InvalidKernelParam;
+
if (PT->isDependentType())
return InvalidKernelParam;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index c070010c24b49..3ec349af59576 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -126,15 +126,21 @@ kernel void test_load_local(__local float *ptr) {
// Same matrix type, __global vs __local -> two distinct symbols.
// ===========================================================================
-kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+kernel void test_store_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
// CHECK-SAME: ptr addrspace(1)
-kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+kernel void test_store_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_init(0.0f);
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 28f7660957a62..112a4d58edab4 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -38,8 +38,10 @@ kernel void test_load(__global float *ptr) {
// ---------------------------------------------------------------------------
// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
-kernel void test_store(__global float *ptr, MatA_t a) {
+kernel void test_store(__global float *ptr) {
+ MatA_t a;
coop_mat_store(ptr, a, ROW_MAJOR, 16);
+ (void)a;
}
// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index 38d98b28482bf..a7737725e5dc7 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -41,7 +41,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
// that contains the vendor-extended "coop_mat" marker.
-kernel void test_mangling(MatA_t a) { (void)a; }
+void test_mangling(MatA_t a) { (void)a; }
// MANGLE: @{{.*}}test_mangling{{.*}}(
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 351728d245494..9e7c992a63a25 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -19,7 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expecte
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t; // expected-note {{declared here}}
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
@@ -65,3 +65,8 @@ kernel void test_bad_assignment(__global float *ptr) {
bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
(void)bad;
}
+
+// ---------------------------------------------------------------------------
+// 5. Kernel parameter cannot be of cooperative matrix type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
>From 8718558f12acd243a069c0e0ab26261a5f74050a Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 23 Sep 2026 19:51:06 -0400
Subject: [PATCH 27/43] Update IsCoopMatBuiltin
---
clang/include/clang/Sema/Sema.h | 2 +-
clang/lib/Sema/SemaDecl.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
4 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 7d00a598ea2e7..2b12d11e9bb32 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15444,7 +15444,7 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
- bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+ bool BuiltinReturnsCoopMatrix(Expr *RHSExpr);
/// Build a function type.
///
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 19620ebd70da4..12670978778af 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14105,8 +14105,8 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if RHSExpr is a cooperative matrix builtin call.
-bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+// Return true if cooperative matrix builtin call returns cooperative matrix.
+bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
auto call = dyn_cast<CallExpr>(RHSExpr);
if (!call)
return false;
@@ -14119,7 +14119,6 @@ bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
- case Builtin::BIcoop_mat_store:
case Builtin::BIcoop_mat_mulAdd:
case Builtin::BIcoop_mat_binary_add:
case Builtin::BIcoop_mat_binary_sub:
@@ -14657,7 +14656,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
if (!VDecl->getType()->isCooperativeMatrixType()) {
Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
return;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 872df233400ba..6ac28d166042b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15998,7 +15998,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
- if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(RHSExpr)) {
if (!LHSExpr->getType()->isCooperativeMatrixType()) {
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index dafa5b3bf123a..5de358100a560 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -25,7 +25,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
// match the LHS variable type. Declare first, then assign so that
-// Sema's IsCoopMatrixBuiltin path fires correctly.
+// Sema's BuiltinReturnsCoopMatrix path fires correctly.
// ---------------------------------------------------------------------------
kernel void test_load_store(__global float *ptr,
__global float *out_ptr) {
>From 82bed6f498a853c2f2f8e13525af839bca80d122 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 16:45:02 -0400
Subject: [PATCH 28/43] Remove dimension check for cooperative matrix
---
clang/include/clang/AST/TypeBase.h | 12 ------------
clang/lib/AST/ASTContext.cpp | 3 ---
2 files changed, 15 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 5328e2db64d2c..b3437f3e7dae6 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4598,8 +4598,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
unsigned Scope;
unsigned Use;
- static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
-
CooperativeMatrixType(QualType ElementType, unsigned Scope, unsigned NumRows,
unsigned NumColumns, unsigned Use,
QualType CanonicalType)
@@ -4624,11 +4622,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns the cooperative matrix use.
unsigned getUse() const { return Use; }
- /// Returns true if \p NumElements is a valid cooperative matrix dimension.
- static constexpr bool isDimensionValid(size_t NumElements) {
- return NumElements > 0 && NumElements <= MaxElementsPerDimension;
- }
-
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
@@ -4641,11 +4634,6 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
Use == 2; // CLK_COOPERATIVE_MATRIX_ACCUMULATOR
}
- /// Returns the maximum valid number of elements per dimension.
- static constexpr unsigned getMaxElementsPerDimension() {
- return MaxElementsPerDimension;
- }
-
void Profile(llvm::FoldingSetNodeID &ID) {
Profile(ID, getElementType(), getScope(), getNumRows(), getNumColumns(),
getUse(), getTypeClass());
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index c1086d2ed5e4e..4618311a0cba8 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -4951,9 +4951,6 @@ QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
"need a valid element type");
- assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
- CooperativeMatrixType::isDimensionValid(NumColumns) &&
- "need valid matrix dimensions");
assert(CooperativeMatrixType::isScopeValid(Scope) &&
"need valid matrix scope");
assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
>From 9ea10e594f30d4303c5756ccd83213d1d1c4f128 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 17:22:16 -0400
Subject: [PATCH 29/43] Update support for 'scope'
---
clang/include/clang/AST/TypeBase.h | 2 +-
clang/include/clang/Basic/AttrDocs.td | 6 ++--
clang/lib/Headers/opencl-c-base.h | 4 ---
.../coop-mat-codegen-mangling.cl | 32 +++++++++----------
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../coop-mat-init-length-codegen.cl | 6 ++--
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 2 +-
.../SemaOpenCL/coop-mat-init-length-sema.cl | 8 ++---
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 4 +--
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 2 +-
11 files changed, 33 insertions(+), 37 deletions(-)
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index b3437f3e7dae6..957149d715fd3 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4624,7 +4624,7 @@ class CooperativeMatrixType final : public Type, public llvm::FoldingSetNode {
/// Returns true if \p Scope is a valid cooperative matrix scope.
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3; // CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+ return Scope == 4; // memory_scope_sub_group
}
/// Returns true if \p Use is a valid cooperative matrix use.
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 93174cae1d539..9023b2f594ebb 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1494,7 +1494,7 @@ arguments:
__attribute__((coop_mat(scope, rows, columns, use)))
```
- scope specifies the memory scope of the cooperative matrix.
- It must be one of the supported OpenCL memory scopes, such as CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP.
+ Only one of the OpenCL memory scopes (memory_scope_sub_group) is supported for cooperative matrix.
- rows specifies the number of rows in the matrix. It must be a constant expression.
- columns specifies the number of columns in the matrix. It must be a constant expression.
- use specifies the operand role of the matrix in coop_mat_muladd. It must be one of:
@@ -1506,13 +1506,13 @@ The coop_mat attribute can be used to declare local variables, function paramete
```c
float vmat1 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
8,
8,
CLK_COOPERATIVE_MATRIX_A)));
typedef half chmat44 __attribute__((coop_mat(
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ memory_scope_sub_group,
4,
4,
CLK_COOPERATIVE_MATRIX_B)));
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 83bb35f1d886a..c4263b7865255 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -632,10 +632,6 @@ CLINKAGE int printf(__constant const char *st, ...)
#undef CLINKAGE
#endif
-typedef enum coop_matrix_scope_t {
- CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
-} coop_matrix_scope_t;
-
typedef enum coop_matrix_operands_t {
CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 3ec349af59576..2446e41d4aec8 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -13,7 +13,7 @@
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -165,18 +165,18 @@ kernel void test_no_cross_contamination(__global float *gptr,
// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
// Verify distinct declare lines exist (these appear at end of IR).
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
-// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_16x16_u0_f32_sc{{[0-9]+}}_16x16_u1_f32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc{{[0-9]+}}_8x32_u0_f32_sc{{[0-9]+}}_32x8_u1_f32_sc{{[0-9]+}}_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc{{[0-9]+}}_16x16_u0_i32_sc{{[0-9]+}}_16x16_u1_i32_sc{{[0-9]+}}_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 112a4d58edab4..0bcb3d1d2ee37 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -8,7 +8,7 @@
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 0b7a825b5d369..ef61b8b40fbbf 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -10,13 +10,13 @@
// ---------------------------------------------------------------------------
// Type definitions.
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index a7737725e5dc7..f16682934759b 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -28,7 +28,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index b32f6549deba5..a9ea17abb2730 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -17,16 +17,16 @@
// ---------------------------------------------------------------------------
#pragma OPENCL EXTENSION cl_khr_cooperative_matrix : enable
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_B))) MatB_t;
-typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef float __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
-typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+typedef int __attribute__((coop_mat(memory_scope_sub_group,
16, 16,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9e7c992a63a25..9315f3cfeb953 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -6,7 +6,7 @@
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
@@ -24,7 +24,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
// ---------------------------------------------------------------------------
-// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// 1. Invalid scope value (0 is not memory_scope_sub_group)
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index 5de358100a560..7bf1f5d2ae184 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -10,7 +10,7 @@
// expected-no-diagnostics
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 3f8b221c3edeb..7f7daab56ebcf 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -24,7 +24,7 @@
// ---------------------------------------------------------------------------
// Enum constants (from opencl-c-base.h via -finclude-default-header)
// ---------------------------------------------------------------------------
-#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
>From 9398a7effc1c8023cdea750df8fb2e77add29eec Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:39:49 -0400
Subject: [PATCH 30/43] Update Semantic analysis to add missing checks
---
.../clang/Basic/DiagnosticSemaKinds.td | 11 +--
clang/include/clang/Sema/Sema.h | 10 +-
clang/lib/Sema/SemaChecking.cpp | 93 ++++++++++++++-----
clang/lib/Sema/SemaDecl.cpp | 14 +--
clang/lib/Sema/SemaExpr.cpp | 19 ++--
5 files changed, 90 insertions(+), 57 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6612665163b67..694db6e5726f8 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11995,13 +11995,6 @@ def ext_coopmat_without_pragma
def err_invalid_coopmat_attr
: Error<"invalid argument of cooperative matrix attribute">;
-def err_coop_mat_not_primitive
- : Error<"cooperative matrix attribute only supported for integer and "
- "floating-point types">;
-
-def err_coop_mat_rol_col_size
- : Error<"cooperative matrix row and column must have power-of-two size">;
-
def err_coop_mem_layout_enum
: Error<"memory layout for cooperative matrix should be "
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
@@ -12052,8 +12045,8 @@ def err_unsupported_coopmat_binary_operator
def err_unsupported_coopmat_scalar_operator
: Error<"unsupported cooperative matrix scalar operator">;
-def err_ref_coopmat_var
- : Error<"invalid reference of cooperative matrix variable">;
+def err_invalid_operand_for_coopmat_scalar_operator
+ : Error<"unsupported type for cooperative matrix scalar operator">;
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 2b12d11e9bb32..bf04230f88d45 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3113,17 +3113,17 @@ class Sema final : public SemaBase {
bool BuiltinCountedByRef(CallExpr *TheCall);
// Coop matrix handling.
- void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc, CallExpr *call);
- void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ bool CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- bool CheckCoopMatrixLoadStoreStride(Expr *Stride);
+ bool CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx);
ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
- void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ bool CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
@@ -8049,7 +8049,7 @@ class Sema final : public SemaBase {
/// Are the two types cooperative matrix types and do they have the same
/// dimensions i.e. do they have the same number of rows and the same number
/// of columns? Also do they have the same scope and use?
- bool areCoopMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
+ bool areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy);
bool areVectorTypesSameSize(QualType srcType, QualType destType);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 4b39ef832298d..4b2a024377c68 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17473,35 +17473,38 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
}
// Check coop_mat_load/store matrix element has same type with buffer pointer.
-void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
QualType BufferType,
SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
- return;
+ return true;
}
assert(isa<PointerType>(BufferType));
auto *PTy = BufferType->castAs<PointerType>();
if (MTy->getElementType().getUnqualifiedType() !=
- PTy->getPointeeType().getUnqualifiedType())
+ PTy->getPointeeType().getUnqualifiedType()) {
Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+ return true;
+ }
+ return false;
}
-void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
SourceLocation MatrixLoc,
- CallExpr *call) {
-
- FunctionDecl *F = call->getDirectCallee();
+ CallExpr *Call) {
+ FunctionDecl *F = Call->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
+ MatrixLoc);
+ return false;
}
// Check coop_mat_load/store layout argument
@@ -17524,11 +17527,14 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(Expr *Stride) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
+ assert(TheCall->getNumArgs() >= ArgIdx + 1);
+ Expr *Stride = TheCall->getArg(ArgIdx);
if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
+ TheCall->setArg(ArgIdx, Stride);
return false;
}
@@ -17540,7 +17546,7 @@ ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(2)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 2 /*ArgIdx*/))
return ExprError();
return CallResult;
}
@@ -17553,23 +17559,24 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
return ExprError();
- CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc()))
+ return ExprError();
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
- if (CheckCoopMatrixLoadStoreStride(TheCall->getArg(3)))
+ if (CheckCoopMatrixLoadStoreStride(TheCall, 3 /*ArgIdx*/))
return ExprError();
return CallResult;
}
-void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
FunctionDecl *F = TheCall->getDirectCallee();
assert(F);
DeclarationName MemberName = F->getDeclName();
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (!Fname->isStr("coop_mat_mulAdd"))
- return;
+ return false;
auto MC = TheCall->getArg(2);
auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
@@ -17581,17 +17588,24 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
if (!M2Ty)
Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
if (!MOutTy || !M2Ty)
- return;
+ return true;
- if (MOutTy->getUse() != 2)
+ if (MOutTy->getUse() != 2) {
Diag(Loc, diag::err_coop_matrix_useACC);
+ return true;
+ }
if (MOutTy->getElementType().getUnqualifiedType() !=
- M2Ty->getElementType().getUnqualifiedType())
+ M2Ty->getElementType().getUnqualifiedType()) {
Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
+ return true;
+ }
- if (!areCoopMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ if (!areCoopMatrixTypesCompatible(TheCall->getType(), MC->getType())) {
Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+ return false;
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
@@ -17605,7 +17619,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (!M0Ty || !M1Ty)
return true;
- if (!areCoopMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ if (!areCoopMatrixTypesCompatible(ATy, BTy)) {
Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
return true;
}
@@ -17660,6 +17674,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ return ExprError();
}
}
@@ -17682,12 +17697,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
if (!M0Ty || !M1Ty || !M2Ty)
return ExprError();
- if (M0Ty->getUse() != 0)
+ if (M0Ty->getUse() != 0) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
- if (M1Ty->getUse() != 1)
+ return ExprError();
+ }
+ if (M1Ty->getUse() != 1) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
- if (M2Ty->getUse() != 2)
+ return ExprError();
+ }
+ if (M2Ty->getUse() != 2) {
Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+ return ExprError();
+ }
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType())
@@ -17712,6 +17733,25 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
+
+ Expr *Arg1 = TheCall->getArg(1);
+ QualType Ty = Arg1->getType();
+ if (!Ty->isScalarType()) {
+ Diag(Arg1->getBeginLoc(),
+ diag::err_invalid_operand_for_coopmat_scalar_operator);
+ return ExprError();
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ Ty.getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
return CallResult;
@@ -17723,6 +17763,11 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
return ExprError();
Expr *Arg0 = TheCall->getArg(0);
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ if (!M0Ty) {
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ return ExprError();
+ }
TheCall->setType(Arg0->getType());
return CallResult;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 12670978778af..3896b6f3019c6 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14105,18 +14105,14 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if cooperative matrix builtin call returns cooperative matrix.
+// Return true if builtin call returns cooperative matrix.
bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
- auto call = dyn_cast<CallExpr>(RHSExpr);
- if (!call)
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ if (!Call)
return false;
- FunctionDecl *F = call->getDirectCallee();
+ FunctionDecl *F = Call->getDirectCallee();
if (!F)
return false;
- DeclarationName MemberName = F->getDeclName();
- IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
- if (!Fname)
- return false;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
case Builtin::BIcoop_mat_mulAdd:
@@ -14131,8 +14127,6 @@ bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
default:
return false;
}
-
- return false;
}
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 6ac28d166042b..553a218beaa4f 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8068,8 +8068,7 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
matSrcType->getNumColumns() == matDestType->getNumColumns();
}
-bool Sema::areCoopMatrixTypesOfTheSameDimension(QualType srcTy,
- QualType destTy) {
+bool Sema::areCoopMatrixTypesCompatible(QualType srcTy, QualType destTy) {
if (!destTy->isCooperativeMatrixType() || !srcTy->isCooperativeMatrixType())
return false;
@@ -8192,7 +8191,7 @@ bool Sema::CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
bool Sema::CheckCoopMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy,
CastKind &Kind) {
if (SrcTy->isCooperativeMatrixType() && DestTy->isCooperativeMatrixType()) {
- if (!areCoopMatrixTypesOfTheSameDimension(SrcTy, DestTy)) {
+ if (!areCoopMatrixTypesCompatible(SrcTy, DestTy)) {
return Diag(R.getBegin(), diag::err_invalid_conversion_between_matrixes)
<< DestTy << SrcTy << R;
}
@@ -16003,12 +16002,14 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
return ExprError();
}
- auto call = dyn_cast<CallExpr>(RHSExpr);
- assert(call);
- call->setType(LHSExpr->getType());
- CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
- CheckCoopMatrixMatMulOutput(call);
+ auto Call = dyn_cast<CallExpr>(RHSExpr);
+ assert(Call);
+ Call->setType(LHSExpr->getType());
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ Call))
+ return ExprError();
+ if (CheckCoopMatrixMatMulOutput(Call))
+ return ExprError();
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
>From 9c49a6313502d938a3e66e89a742fa7fe02767cb Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 24 Sep 2026 20:42:05 -0400
Subject: [PATCH 31/43] Fix clang format issues
---
clang/lib/Sema/SemaChecking.cpp | 7 +++----
clang/lib/Sema/SemaExpr.cpp | 4 ++--
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 4b2a024377c68..c82014e0c820b 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17502,8 +17502,8 @@ bool Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
assert(Fname);
if (Fname->isStr("coop_mat_load"))
- return CheckCoopMatrixLoadStoreElementType(MatrixType, Call->getArg(0)->getType(),
- MatrixLoc);
+ return CheckCoopMatrixLoadStoreElementType(
+ MatrixType, Call->getArg(0)->getType(), MatrixLoc);
return false;
}
@@ -17749,8 +17749,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
}
- if (M0Ty->getElementType().getUnqualifiedType() !=
- Ty.getUnqualifiedType())
+ if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
TheCall->setType(Arg0->getType());
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 553a218beaa4f..32213a64ea708 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16005,8 +16005,8 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
auto Call = dyn_cast<CallExpr>(RHSExpr);
assert(Call);
Call->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- Call))
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
+ LHSExpr->getBeginLoc(), Call))
return ExprError();
if (CheckCoopMatrixMatMulOutput(Call))
return ExprError();
>From 69a813d6a9bf2e931637aa138add04ced9053027 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 14:57:35 -0400
Subject: [PATCH 32/43] Fix issue caused by recent merge to the tip.
---
clang/lib/Sema/SemaChecking.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index c82014e0c820b..8f00e04385b78 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17530,7 +17530,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
- if (convertArgumentToType(*this, Stride, Context.getSizeType())) {
+ if (convertArgumentToType(Stride, Context.getSizeType())) {
Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
return true;
}
>From 8128fbf3fda7d81269b2fb728dac174bb592774b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 16:35:51 -0400
Subject: [PATCH 33/43] Add missing check in TypePrinter.cpp
---
clang/lib/AST/TypePrinter.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 34380e3c8965f..047ebdc71d7d9 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2178,6 +2178,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
case attr::ExtVectorType:
OS << "ext_vector_type";
break;
+ case attr::CoopMatrixType:
+ OS << "coop_mat";
+ break;
case attr::CFISalt:
OS << "cfi_salt(\"" << cast<CFISaltAttr>(T->getAttr())->getSalt() << "\")";
break;
>From 0d172a642586deb373aa2aaba9e15945af7048a8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Mon, 28 Sep 2026 18:07:29 -0400
Subject: [PATCH 34/43] Update test
---
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index f77fe542fd210..10fd9c08173b5 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -31,10 +31,6 @@
// expected-no-diagnostics
void test_enum_constants(void) {
- // coop_matrix_scope_t
- coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
- (void)s;
-
// coop_matrix_use_t
coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
@@ -53,7 +49,7 @@ void test_enum_constants(void) {
// ── 2d. Enum constant values match the spec ──────────────────────────────────
void test_enum_values(void) {
- _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(memory_scope_sub_group == 4, "scope subgroup");
_Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
_Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
_Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
>From 2f0e229c6d8eade085f223d91c3361e64adef0e7 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 29 Sep 2026 12:37:47 -0400
Subject: [PATCH 35/43] Fix minor build issue
---
clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index d89c80ca01754..7a0b81494cf1e 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -488,6 +488,7 @@ void ExprEngine::VisitCastExpr(const CastExpr *CastE, ExplodedNode *Pred,
// Various C++ casts that are not handled yet.
case CK_ToUnion:
case CK_MatrixCast:
+ case CK_CoopMatrixCast:
case CK_VectorSplat:
case CK_HLSLElementwiseCast:
case CK_HLSLAggregateSplatCast:
>From 9a8fdf33d6af3864c72c0cddb1970cf3545c1cfc Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:06:37 -0400
Subject: [PATCH 36/43] Fix minor merge issue
---
clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index f7202caf8e5f3..f7fbd0a42e7a2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -2341,9 +2341,6 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
case Builtin::BIcoop_mat_scalar_neg:
case Builtin::BIcoop_mat_init:
case Builtin::BIcoop_mat_length:
- case Builtin::BI__builtin_matrix_transpose:
- case Builtin::BI__builtin_matrix_column_major_load:
- case Builtin::BI__builtin_matrix_column_major_store:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
>From e3f0ee4d8895c609364eef2631efba269cc5ae19 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:07:46 -0400
Subject: [PATCH 37/43] Fix issue of direct initialization of cooperative
matrix
---
clang/lib/Sema/SemaDecl.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 3896b6f3019c6..8c6e7d709d0e0 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14345,6 +14345,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
Init = Result.get();
}
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
+
// Perform the initialization.
bool InitializedFromParenListExpr = false;
bool IsParenListInit = false;
@@ -14647,19 +14660,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
-
- // Set return type of builtin call using type of LHS variable.
- // This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
- if (!VDecl->getType()->isCooperativeMatrixType()) {
- Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
- return;
- }
-
- auto call = dyn_cast<CallExpr>(Init);
- assert(call);
- call->setType(VDecl->getType());
- }
}
void Sema::ActOnInitializerError(Decl *D) {
>From 2c9dc2cd333a58a168991532820030106198307b Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 15:43:36 -0400
Subject: [PATCH 38/43] Update correct emission of MatrixOperand operand for
coop_mat_mulAdd
---
clang/include/clang/Basic/AttrDocs.td | 2 +-
clang/lib/CodeGen/CGBuiltin.cpp | 16 ++++++----------
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 9023b2f594ebb..b9eb5606319a8 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1521,7 +1521,7 @@ chmat44 vmat2;
The base component type must be a scalar numerical type.
Implicit and explicit casts between cooperative matrix types are not generally permitted.
Conversion between cooperative matrix types is allowed only when the types have identical scope, dimensions, and use parameters, and their component types are compatible according to the OpenCL C conversion rules.
-Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_matmul_add, and coop_mat_length.
+Cooperative matrix types are intended to be used with the cooperative matrix functions, including coop_mat_load, coop_mat_store, coop_mat_mulAdd, coop_mat_length and element-wise arithmetic operations.
See the OpenCL documentation for some more complete examples.
}];
}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c3c18e616218b..47c286f16f9b1 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -4795,27 +4795,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
auto Arg2 = E->getArg(2);
+ Value *Zero = Constant::getNullValue(Int32Ty);
auto MA = EmitScalarExpr(Arg0);
auto MB = EmitScalarExpr(Arg1);
auto MC = EmitScalarExpr(Arg2);
+ auto MatrixOperand =
+ (E->getNumArgs() == 4) ? EmitScalarExpr(E->getArg(3)) : Zero;
+
const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
- // Check if the data is signed/unsigned
- QualType QT = MATy->getElementType();
- bool isSigned = false;
- if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
- isSigned = BT->isSignedInteger();
- }
- llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
- llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
auto *RetType = getTargetExtType(CGM, MTy);
auto *AType = getTargetExtType(CGM, MATy);
auto *BType = getTargetExtType(CGM, MBTy);
auto *CType = getTargetExtType(CGM, MCTy);
llvm::FunctionType *FTy =
- llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int32Ty}, false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
@@ -4826,7 +4822,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
- auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, MatrixOperand});
NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
return RValue::get(NewCall);
}
>From 1be787b9af7d251c106be0e68ccf586a49829508 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 16:43:35 -0400
Subject: [PATCH 39/43] Remove unrelated verifier test and update some of the
sema checks.
---
clang/lib/Sema/SemaChecking.cpp | 15 ++++++--
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ---------------------
2 files changed, 12 insertions(+), 42 deletions(-)
delete mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 8f00e04385b78..c6fa96c5491d4 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17672,7 +17672,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
Expr *Operands = TheCall->getArg(3);
QualType OperandsTy = Operands->getType().getCanonicalType();
- if (!OperandsTy->isEnumeralType()) {
+ if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
return ExprError();
}
@@ -17710,9 +17710,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
}
- if (M0Ty->getElementType().getUnqualifiedType() !=
- M1Ty->getElementType().getUnqualifiedType())
+ QualType M0ElemTy =
+ M0Ty->getElementType().getUnqualifiedType().getCanonicalType();
+ QualType M1ElemTy =
+ M1Ty->getElementType().getUnqualifiedType().getCanonicalType();
+ bool SameElementType = M0ElemTy == M1ElemTy;
+ bool SameIntegerWidth =
+ M0ElemTy->isIntegerType() && M1ElemTy->isIntegerType() &&
+ Context.getTypeSize(M0ElemTy) == Context.getTypeSize(M1ElemTy);
+
+ if (!SameElementType && !SameIntegerWidth) {
return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ }
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
deleted file mode 100644
index 7f2eedb657d41..0000000000000
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; llvm/test/Verifier/coop_mat_verifier_fix.ll
-;
-; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
-;
-; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
-; ((A && B) <= 64). After the fix the parentheses are explicit.
-;
-; This test verifies that a valid @llvm.experimental.noalias.scope.decl
-; call with a 2-element scope list (no third argument) passes the verifier
-; without error, and that an invalid third argument (> 64 bits) is caught.
-;
-; ── 6a. Valid call — no third argument → verifier must accept ────────────────
-; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
-; RUN: llvm-as < %s | opt -passes=verify -disable-output
-;
-; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
-; RUN: llvm-as %s -o %t.bc
-; RUN: opt -passes=verify %t.bc -disable-output
-
-; VALID: @llvm.experimental.noalias.scope.decl
-
-declare void @llvm.experimental.noalias.scope.decl(metadata)
-
-define void @test_no_third_arg() {
- call void @llvm.experimental.noalias.scope.decl(
- metadata !0)
- ret void
-}
-
-; Third arg present and <= 64 bits wide — must also pass
-define void @test_valid_third_arg() {
- call void @llvm.experimental.noalias.scope.decl(
- metadata !0)
- ret void
-}
-
-!0 = !{!1}
-!1 = distinct !{!1, !2, !"scope_a"}
-!2 = distinct !{!2, !"domain_a"}
>From c9873180ace0b63fac18891d94a231578f60ee80 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 7 Oct 2026 19:38:35 -0400
Subject: [PATCH 40/43] Update more sema checks
---
.../clang/Basic/DiagnosticSemaKinds.td | 8 +++---
clang/include/clang/Sema/Sema.h | 2 +-
clang/lib/Sema/SemaChecking.cpp | 9 ++----
clang/lib/Sema/SemaDecl.cpp | 28 +++++++++----------
clang/lib/Sema/SemaExpr.cpp | 24 ++++++++--------
5 files changed, 33 insertions(+), 38 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 694db6e5726f8..dd3699f7f646b 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12014,7 +12014,7 @@ def err_coop_matrix_arg
def err_coop_matrix_operands_type
: Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
-def err_coop_matrix_assignment
+def err_invalid_coopmat_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
@@ -12036,9 +12036,6 @@ def err_invalid_coop_matrix_element_type
def err_coop_matrix_row_or_col_mismatch
: Error<"mismatch of cooperative matrix row or column">;
-def err_coop_matrix_use_type
- : Error<"inconsistent cooperative matrix use type">;
-
def err_unsupported_coopmat_binary_operator
: Error<"unsupported cooperative matrix binary operator">;
@@ -12048,6 +12045,9 @@ def err_unsupported_coopmat_scalar_operator
def err_invalid_operand_for_coopmat_scalar_operator
: Error<"unsupported type for cooperative matrix scalar operator">;
+def err_incompatible_coopmat_types
+ : Error<"incompatible cooperative matrix types">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index bf04230f88d45..da9df21670832 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15444,7 +15444,7 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
- bool BuiltinReturnsCoopMatrix(Expr *RHSExpr);
+ CallExpr *getCoopMatrixBuiltinCall(Expr *RHSExpr);
/// Build a function type.
///
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index c6fa96c5491d4..e924fc002871f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17602,7 +17602,7 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
if (!areCoopMatrixTypesCompatible(TheCall->getType(), MC->getType())) {
- Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+ Diag(Loc, diag::err_incompatible_coopmat_types);
return true;
}
return false;
@@ -17620,12 +17620,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
return true;
if (!areCoopMatrixTypesCompatible(ATy, BTy)) {
- Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
- return true;
- }
-
- if (M0Ty->getUse() != M1Ty->getUse()) {
- Diag(ALoc, diag::err_coop_matrix_use_type);
+ Diag(ALoc, diag::err_incompatible_coopmat_types);
return true;
}
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 8c6e7d709d0e0..4624a5ece5b00 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -14105,14 +14105,15 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
-// Return true if builtin call returns cooperative matrix.
-bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
+// Return the CallExpr if builtin call returns cooperative matrix.
+CallExpr *Sema::getCoopMatrixBuiltinCall(Expr *RHSExpr) {
+ RHSExpr = RHSExpr->IgnoreParens();
auto Call = dyn_cast<CallExpr>(RHSExpr);
if (!Call)
- return false;
+ return nullptr;
FunctionDecl *F = Call->getDirectCallee();
if (!F)
- return false;
+ return nullptr;
switch (F->getBuiltinID()) {
case Builtin::BIcoop_mat_load:
case Builtin::BIcoop_mat_mulAdd:
@@ -14123,9 +14124,9 @@ bool Sema::BuiltinReturnsCoopMatrix(Expr *RHSExpr) {
case Builtin::BIcoop_mat_scalar_mul:
case Builtin::BIcoop_mat_scalar_neg:
case Builtin::BIcoop_mat_init:
- return true;
+ return Call;
default:
- return false;
+ return nullptr;
}
}
@@ -14347,15 +14348,14 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
// Set return type of builtin call using type of LHS variable.
// This is done for builtin calls that return cooperative matrix.
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(Init)) {
- if (!VDecl->getType()->isCooperativeMatrixType()) {
- Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
- return;
+ if (getLangOpts().OpenCL) {
+ if (CallExpr *Call = getCoopMatrixBuiltinCall(Init)) {
+ if (!VDecl->getType()->isCooperativeMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_invalid_coopmat_assignment);
+ return;
+ }
+ Call->setType(VDecl->getType());
}
-
- auto call = dyn_cast<CallExpr>(Init);
- assert(call);
- call->setType(VDecl->getType());
}
// Perform the initialization.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 32213a64ea708..4d46a1244f130 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -15997,19 +15997,19 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
- if (getLangOpts().OpenCL && BuiltinReturnsCoopMatrix(RHSExpr)) {
- if (!LHSExpr->getType()->isCooperativeMatrixType()) {
- Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
- return ExprError();
+ if (getLangOpts().OpenCL) {
+ if (CallExpr *Call = getCoopMatrixBuiltinCall(RHSExpr)) {
+ if (!LHSExpr->getType()->isCooperativeMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_invalid_coopmat_assignment);
+ return ExprError();
+ }
+ Call->setType(LHSExpr->getType());
+ if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
+ LHSExpr->getBeginLoc(), Call))
+ return ExprError();
+ if (CheckCoopMatrixMatMulOutput(Call))
+ return ExprError();
}
- auto Call = dyn_cast<CallExpr>(RHSExpr);
- assert(Call);
- Call->setType(LHSExpr->getType());
- if (CheckCoopMatrixLoadElementType(LHSExpr->getType(),
- LHSExpr->getBeginLoc(), Call))
- return ExprError();
- if (CheckCoopMatrixMatMulOutput(Call))
- return ExprError();
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
>From edac5865d82bdfb5f377f21be3efef224337c366 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 02:00:25 -0400
Subject: [PATCH 41/43] Update diagnostics and their use in Semantic analysis
and the associated tests
---
.../clang/Basic/DiagnosticSemaKinds.td | 40 ++--
clang/lib/Sema/SemaChecking.cpp | 73 +++----
clang/lib/Sema/SemaType.cpp | 8 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 178 ++++++++++++++++--
4 files changed, 219 insertions(+), 80 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index dd3699f7f646b..29f17ce58163e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11989,52 +11989,45 @@ def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
// OpenCL C Cooperative Matrix.
-def ext_coopmat_without_pragma
+def err_coopmat_without_pragma
: Error<"cooperative matrix types require OpenCL extension '%0' to be enabled via pragma">;
def err_invalid_coopmat_attr
- : Error<"invalid argument of cooperative matrix attribute">;
+ : Error<"invalid '%0' argument of cooperative matrix attribute">;
-def err_coop_mem_layout_enum
- : Error<"memory layout for cooperative matrix should be "
+def err_invalid_coopmat_mem_layout
+ : Error<"memory layout for cooperative matrix load/store should be "
"CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
"CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
-def err_coop_mat_stride_type
+def err_invalid_coopmat_stride_type
: Error<"stride argument for load/store of cooperative matrix must have "
"type 'size_t'">;
-def err_coop_element_and_pointer_type
+def err_coopmat_mismatch_element_and_pointer_type
: Error<"inconsistent between cooperative matrix element type and buffer "
"pointer type">;
-def err_coop_matrix_arg
+def err_invalid_coopmat_arg
: Error<"argument must be a valid cooperative matrix type">;
-def err_coop_matrix_operands_type
+def err_invalid_coopmat_memory_operand_type
: Error<"fourth argument of 'coop_mat_mulAdd' must have enum type">;
def err_invalid_coopmat_assignment
: Error<"builtin return value should be assigned to cooperative matrix "
"type variable">;
-def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_A">;
+def err_invalid_coopmat_use : Error<"argument of cooperative matrix must have '%0' use">;
-def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_B">;
+def err_mismatched_coopmat_element_types
+ : Error<"inconsistent cooperative matrix element types">;
-def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
- "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+def err_invalid_coopmat_element_type
+ : Error<"%0 is invalid cooperative matrix element type">;
-def err_mismatched_coop_matrix_element_type
- : Error<"inconsistent cooperative matrix element type">;
-
-def err_invalid_coop_matrix_element_type
- : Error<"invalid cooperative matrix element type">;
-
-def err_coop_matrix_row_or_col_mismatch
- : Error<"mismatch of cooperative matrix row or column">;
+def err_coopmat_shapes_incompatible
+ : Error<"two cooperative matrices with incompatible shapes">;
def err_unsupported_coopmat_binary_operator
: Error<"unsupported cooperative matrix binary operator">;
@@ -12042,9 +12035,6 @@ def err_unsupported_coopmat_binary_operator
def err_unsupported_coopmat_scalar_operator
: Error<"unsupported cooperative matrix scalar operator">;
-def err_invalid_operand_for_coopmat_scalar_operator
- : Error<"unsupported type for cooperative matrix scalar operator">;
-
def err_incompatible_coopmat_types
: Error<"incompatible cooperative matrix types">;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e924fc002871f..f7dbf83755cc4 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17478,7 +17478,7 @@ bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
- Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ Diag(MatrixLoc, diag::err_invalid_coopmat_arg);
return true;
}
@@ -17487,7 +17487,7 @@ bool Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
if (MTy->getElementType().getUnqualifiedType() !=
PTy->getPointeeType().getUnqualifiedType()) {
- Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+ Diag(MatrixLoc, diag::err_coopmat_mismatch_element_and_pointer_type);
return true;
}
return false;
@@ -17522,7 +17522,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
ArgError = true;
if (ArgError)
- Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+ Diag(LayoutExpr->getBeginLoc(), diag::err_invalid_coopmat_mem_layout);
return ArgError;
}
@@ -17530,8 +17530,16 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
- if (convertArgumentToType(Stride, Context.getSizeType())) {
- Diag(Stride->getExprLoc(), diag::err_coop_mat_stride_type);
+ QualType StrideType = Stride->getType();
+ QualType SizeType = Context.getSizeType();
+ bool IsSizeT = Context.hasSameType(StrideType, SizeType);
+ bool IsIntegerConstant = Stride->isIntegerConstantExpr(Context);
+ if (!IsSizeT && !IsIntegerConstant) {
+ Diag(Stride->getExprLoc(), diag::err_invalid_coopmat_stride_type);
+ return true;
+ }
+ if (convertArgumentToType(Stride, SizeType)) {
+ Diag(Stride->getExprLoc(), diag::err_invalid_coopmat_stride_type);
return true;
}
TheCall->setArg(ArgIdx, Stride);
@@ -17584,20 +17592,21 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
auto Loc = TheCall->getBeginLoc();
if (!MOutTy)
- Diag(Loc, diag::err_coop_matrix_arg);
+ Diag(Loc, diag::err_invalid_coopmat_arg);
if (!M2Ty)
- Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(MC->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!MOutTy || !M2Ty)
return true;
if (MOutTy->getUse() != 2) {
- Diag(Loc, diag::err_coop_matrix_useACC);
+ Diag(Loc, diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return true;
}
if (MOutTy->getElementType().getUnqualifiedType() !=
M2Ty->getElementType().getUnqualifiedType()) {
- Diag(Loc, diag::err_mismatched_coop_matrix_element_type);
+ Diag(Loc, diag::err_mismatched_coopmat_element_types);
return true;
}
@@ -17613,9 +17622,9 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
- Diag(ALoc, diag::err_coop_matrix_arg);
+ Diag(ALoc, diag::err_invalid_coopmat_arg);
if (!M1Ty)
- Diag(BLoc, diag::err_coop_matrix_arg);
+ Diag(BLoc, diag::err_invalid_coopmat_arg);
if (!M0Ty || !M1Ty)
return true;
@@ -17626,7 +17635,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
if (M0Ty->getElementType().getUnqualifiedType() !=
M1Ty->getElementType().getUnqualifiedType()) {
- Diag(ALoc, diag::err_mismatched_coop_matrix_element_type);
+ Diag(ALoc, diag::err_mismatched_coopmat_element_types);
return true;
}
return false;
@@ -17668,7 +17677,8 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
QualType OperandsTy = Operands->getType().getCanonicalType();
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
- Diag(Operands->getBeginLoc(), diag::err_coop_matrix_operands_type);
+ Diag(Operands->getBeginLoc(),
+ diag::err_invalid_coopmat_memory_operand_type);
return ExprError();
}
}
@@ -17684,24 +17694,27 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
auto Loc1 = Arg1->getBeginLoc();
if (!M0Ty)
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M1Ty)
- Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg1->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M2Ty)
- Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg2->getBeginLoc(), diag::err_invalid_coopmat_arg);
if (!M0Ty || !M1Ty || !M2Ty)
return ExprError();
if (M0Ty->getUse() != 0) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_A";
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
+ << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
@@ -17715,18 +17728,18 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
Context.getTypeSize(M0ElemTy) == Context.getTypeSize(M1ElemTy);
if (!SameElementType && !SameIntegerWidth) {
- return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
}
if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
M2Ty->getElementType()))
- return ExprError(Diag(Loc1, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc1, diag::err_mismatched_coopmat_element_types));
if (M0Ty->getNumRows() != M2Ty->getNumRows())
- return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ return ExprError(Diag(Loc0, diag::err_coopmat_shapes_incompatible));
if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
(M0Ty->getNumColumns() != M1Ty->getNumRows()))
- return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+ return ExprError(Diag(Loc1, diag::err_coopmat_shapes_incompatible));
return CallResult;
}
@@ -17741,20 +17754,12 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
auto Loc0 = Arg0->getBeginLoc();
if (!M0Ty) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
- return ExprError();
- }
-
- Expr *Arg1 = TheCall->getArg(1);
- QualType Ty = Arg1->getType();
- if (!Ty->isScalarType()) {
- Diag(Arg1->getBeginLoc(),
- diag::err_invalid_operand_for_coopmat_scalar_operator);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
return ExprError();
}
if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
- return ExprError(Diag(Loc0, diag::err_mismatched_coop_matrix_element_type));
+ return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
TheCall->setType(Arg0->getType());
return CallResult;
@@ -17768,7 +17773,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
Expr *Arg0 = TheCall->getArg(0);
auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
if (!M0Ty) {
- Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_arg);
return ExprError();
}
TheCall->setType(Arg0->getType());
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 8bc9451365c06..cbfe4fbec0e26 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2683,15 +2683,15 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
if (!CooperativeMatrixType::isScopeValid(Scope)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Scope << "matrix scope";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << "scope";
return QualType();
}
if (!CooperativeMatrixType::isUseValid(Use)) {
- Diag(AttrLoc, diag::err_invalid_coopmat_attr) << Use << "matrix use";
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << "use";
return QualType();
}
if (!CooperativeMatrixType::isValidElementType(ElementTy)) {
- Diag(AttrLoc, diag::err_invalid_coop_matrix_element_type) << ElementTy;
+ Diag(AttrLoc, diag::err_invalid_coopmat_element_type) << ElementTy;
return QualType();
}
auto Result = Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
@@ -2703,7 +2703,7 @@ QualType Sema::BuildCoopMatrixType(QualType ElementTy, Expr *ScopeExpr,
<< 0 << Result << "cl_khr_cooperative_matrix";
else if (!getOpenCLOptions().isAvailableOption("cl_khr_cooperative_matrix",
getLangOpts()))
- Diag(AttrLoc, diag::ext_coopmat_without_pragma)
+ Diag(AttrLoc, diag::err_coopmat_without_pragma)
<< "cl_khr_cooperative_matrix";
}
return Result;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index 9315f3cfeb953..d7b61d5d57b18 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -1,6 +1,4 @@
-// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
-//
-// Patch 4: Sema — negative tests for diagnostic paths.
+// Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -13,7 +11,7 @@
#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
// ---------------------------------------------------------------------------
-// 0. Use of coop mat type without specifying pragma
+// Use of cooperative matrix type without specifying pragma
// ---------------------------------------------------------------------------
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatX_t; // expected-error {{cooperative matrix types require OpenCL extension 'cl_khr_cooperative_matrix' to be enabled via pragma}}
@@ -24,21 +22,19 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
// ---------------------------------------------------------------------------
-// 1. Invalid scope value (0 is not memory_scope_sub_group)
+// Invalid scope value (0 is not memory_scope_sub_group)
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid 'scope' argument of cooperative matrix attribute}}
// ---------------------------------------------------------------------------
-// 2. Invalid use value (99 is not 0/1/2)
+// Invalid use value (99 is not 0/1/2)
// ---------------------------------------------------------------------------
-typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid 'use' argument of cooperative matrix attribute}}
// ---------------------------------------------------------------------------
-// 3. Mismatched element types in coop_mat_mulAdd
+// Mismatched element types in coop_mat_mulAdd
// a/b are float matrices, c is an int matrix — should fire element type
// mismatch diagnostic.
-// Use the two-step declare-then-assign pattern so that the coop_mat_load
-// fixup path fires correctly and we only get the intended mulAdd error.
// ---------------------------------------------------------------------------
kernel void test_muladd_type_mismatch(__global float *fptr,
__global int *iptr) {
@@ -50,15 +46,13 @@ kernel void test_muladd_type_mismatch(__global float *fptr,
c = coop_mat_load(iptr, ROW_MAJOR, 16);
MatC_int_t result;
- result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element types}}
(void)result;
}
// ---------------------------------------------------------------------------
-// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
-// "should be assigned to cooperative matrix type variable" diagnostic.
-// This intentionally uses single-step initialization because the error
-// fires precisely when the LHS is NOT a coop mat type.
+// Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
// ---------------------------------------------------------------------------
kernel void test_bad_assignment(__global float *ptr) {
float bad;
@@ -67,6 +61,156 @@ kernel void test_bad_assignment(__global float *ptr) {
}
// ---------------------------------------------------------------------------
-// 5. Kernel parameter cannot be of cooperative matrix type.
+// Kernel parameter cannot be of cooperative matrix type.
// ---------------------------------------------------------------------------
kernel void test_bad_kernel_param(MatA_t a) {} // expected-error {{cannot be used as the type of a kernel parameter}}
+
+// ---------------------------------------------------------------------------
+// Invalid memory layout for cooperative matrix load.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_load_layout(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid memory layout for cooperative matrix store.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_store_layout(__global float *ptr) {
+ MatA_t a;
+ coop_mat_store(ptr, a, 99, 16); // expected-error {{memory layout for cooperative matrix load/store should be CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid stride type for cooperative matrix load.
+// The stride must have type size_t.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_load_stride(__global float *ptr) {
+ MatA_t a;
+ int stride = 16;
+ a = coop_mat_load(ptr, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid stride type for cooperative matrix store.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_store_stride(__global float *ptr) {
+ MatA_t a;
+ int stride = 16;
+ coop_mat_store(ptr, a, ROW_MAJOR, stride); // expected-error {{stride argument for load/store of cooperative matrix must have type 'size_t'}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix element type must match the pointed-to buffer type.
+// ---------------------------------------------------------------------------
+kernel void test_load_element_pointer_mismatch(__global int *iptr) {
+ MatA_t a;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16); // expected-error {{inconsistent between cooperative matrix element type and buffer pointer type}}
+}
+
+// ---------------------------------------------------------------------------
+// The fourth coop_mat_mulAdd argument must have the cooperative-matrix
+// operands enum type.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_memory_operand_type() {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+
+ // The literal has type float rather than the required enum type.
+ (void)coop_mat_mulAdd(a, b, c, 16.2f); // expected-error {{fourth argument of 'coop_mat_mulAdd' must have enum type}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd arguments must be cooperative matrix types.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_argument(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(a, b, 1.0f); // expected-error {{argument must be a valid cooperative matrix type}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the first operand to have use A.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_a_use(__global float *ptr) {
+ MatB_t b;
+ MatB_t b2;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(b, b2, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_A' use}}
+}
+
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the second operand to have use B.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_b_use(__global float *ptr) {
+ MatA_t a;
+ MatA_t a2;
+ MatC_int_t c;
+
+ (void)coop_mat_mulAdd(a, a2, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_B' use}}
+}
+
+// ---------------------------------------------------------------------------
+// coop_mat_mulAdd requires the third operand to have accumulator use.
+// ---------------------------------------------------------------------------
+kernel void test_invalid_muladd_accumulator_use(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatA_t c;
+
+ (void)coop_mat_mulAdd(a, b, c); // expected-error {{argument of cooperative matrix must have 'CLK_COOPERATIVE_MATRIX_ACCUMULATOR' use}}
+}
+
+// ---------------------------------------------------------------------------
+// Invalid cooperative matrix element type.
+// ---------------------------------------------------------------------------
+typedef bool __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatBadElement_t; // expected-error {{'bool' is invalid cooperative matrix element type}}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrices with incompatible shapes cannot be used together.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 8, 16, USE_B))) MatB8x16_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC16x16_t;
+
+kernel void test_incompatible_muladd_shapes() {
+ MatA_t a;
+ MatB8x16_t b;
+ MatC16x16_t c;
+
+ (void)coop_mat_mulAdd(a, b, c); // expected-error {{two cooperative matrices with incompatible shapes}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix binary operators require supported operand types.
+// ---------------------------------------------------------------------------
+kernel void test_unsupported_coopmat_binary_operator() {
+ MatA_t a;
+ MatB_t b;
+
+ (void)(a & b); // expected-error {{unsupported cooperative matrix binary operator}}
+}
+
+// ---------------------------------------------------------------------------
+// Scalar cooperative-matrix operators are restricted to the supported
+// operator/type combinations.
+// ---------------------------------------------------------------------------
+kernel void test_unsupported_coopmat_scalar_operator() {
+ MatA_t a;
+
+ (void)(a / 2.0f); // expected-error {{unsupported cooperative matrix scalar operator}}
+}
+
+// ---------------------------------------------------------------------------
+// Cooperative matrix assignment requires compatible cooperative matrix types.
+// ---------------------------------------------------------------------------
+kernel void test_incompatible_coopmat_assignment() {
+ MatA_t a;
+ MatB_t b;
+
+ a = b; // expected-error {{incompatible cooperative matrix types}}
+}
>From 26a19cd26c1137b5ca81c30e06a2aae6c9b8ad60 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 12:05:11 -0400
Subject: [PATCH 42/43] Update tests
---
clang/test/AST/undocumented-attrs.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index ed82ff3f517fd..525466b1efdc0 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -88,4 +88,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 82
\ No newline at end of file
+CHECK-NEXT: Total: 82
>From 88987bd887b9cb5c2b82f611218194b5b22729f4 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Thu, 8 Oct 2026 13:00:28 -0400
Subject: [PATCH 43/43] Cleanup tests and one clang-format issue fix
---
clang/lib/Sema/SemaChecking.cpp | 20 ++++-----
.../coop-mat-codegen-mangling.cl | 9 +---
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 30 ++++++++-----
.../coop-mat-init-length-codegen.cl | 12 ++---
.../test/Preprocessor/coop-mat-opencl-ext.cl | 17 +++----
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 44 +++----------------
.../SemaOpenCL/coop-mat-init-length-sema.cl | 23 +++-------
clang/test/SemaOpenCL/coop-mat-sema.cl | 21 +++------
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 12 +----
9 files changed, 62 insertions(+), 126 deletions(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index f7dbf83755cc4..5549bc94f7a22 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17527,7 +17527,8 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
return ArgError;
}
-bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall, unsigned ArgIdx) {
+bool Sema::CheckCoopMatrixLoadStoreStride(CallExpr *TheCall,
+ unsigned ArgIdx) {
assert(TheCall->getNumArgs() >= ArgIdx + 1);
Expr *Stride = TheCall->getArg(ArgIdx);
QualType StrideType = Stride->getType();
@@ -17599,8 +17600,7 @@ bool Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
return true;
if (MOutTy->getUse() != 2) {
- Diag(Loc, diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Loc, diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return true;
}
@@ -17677,8 +17677,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
QualType OperandsTy = Operands->getType().getCanonicalType();
if (!OperandsTy->isIntegerType() && !OperandsTy->isEnumeralType()) {
- Diag(Operands->getBeginLoc(),
- diag::err_invalid_coopmat_memory_operand_type);
+ Diag(Operands->getBeginLoc(), diag::err_invalid_coopmat_memory_operand_type);
return ExprError();
}
}
@@ -17703,18 +17702,15 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
return ExprError();
if (M0Ty->getUse() != 0) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_A";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_A";
return ExprError();
}
if (M1Ty->getUse() != 1) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_B";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_B";
return ExprError();
}
if (M2Ty->getUse() != 2) {
- Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use)
- << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
+ Diag(Arg0->getBeginLoc(), diag::err_invalid_coopmat_use) << "CLK_COOPERATIVE_MATRIX_ACCUMULATOR";
return ExprError();
}
@@ -17758,6 +17754,8 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
return ExprError();
}
+ Expr *Arg1 = TheCall->getArg(1);
+ QualType Ty = Arg1->getType();
if (M0Ty->getElementType().getUnqualifiedType() != Ty.getUnqualifiedType())
return ExprError(Diag(Loc0, diag::err_mismatched_coopmat_element_types));
TheCall->setType(Arg0->getType());
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 2446e41d4aec8..3f08145c655e6 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -1,11 +1,4 @@
-// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
-//
-// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
-// Covers three mangling axes:
-// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
-// (B) coop_mat_mulAdd -- different element types (float vs int)
-// (C) coop_mat_load -- different pointer address spaces
-// (D) coop_mat_store -- different pointer address spaces
+// Tests for function-name mangling of SPIR-V friendly cooperative matrix intrinsics.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 0bcb3d1d2ee37..6eafdf44d5075 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -1,7 +1,5 @@
-// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
-//
-// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
-// SPIR-V intrinsic names.
+// Cooperative matrix tests for SPIR-V friendly LLVM IR code generation from
+// clang frontend.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -21,7 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
-// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
// TargetExtType (visible in the call signature).
// ---------------------------------------------------------------------------
@@ -36,7 +34,19 @@ kernel void test_load(__global float *ptr) {
// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
// ---------------------------------------------------------------------------
-// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// Same test as above. But 'a' is initialized directly using the 'load'.
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
kernel void test_store(__global float *ptr) {
MatA_t a;
@@ -47,7 +57,7 @@ kernel void test_store(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
// ---------------------------------------------------------------------------
-// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
// ---------------------------------------------------------------------------
kernel void test_muladd(__global float *ptr) {
MatA_t a; MatB_t b; MatC_t c; MatC_t r;
@@ -61,7 +71,7 @@ kernel void test_muladd(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
// ---------------------------------------------------------------------------
-// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// Binary add (float element) -> __spirv_CooperativeMatrixFAdd
// ---------------------------------------------------------------------------
kernel void test_binary_add(__global float *ptr) {
MatA_t a; MatA_t b; MatA_t r;
@@ -74,7 +84,7 @@ kernel void test_binary_add(__global float *ptr) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
// ---------------------------------------------------------------------------
-// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
kernel void test_scalar_mul(__global float *ptr, float s) {
MatA_t a; MatA_t r;
@@ -90,7 +100,7 @@ kernel void test_scalar_mul(__global float *ptr, float s) {
// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
// ---------------------------------------------------------------------------
-// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// Unary minus -> __spirv_CooperativeMatrixScalarNeg
// ---------------------------------------------------------------------------
kernel void test_unary_neg(__global float *ptr) {
MatA_t a; MatA_t r;
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index ef61b8b40fbbf..b7673d7cdf0f6 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -21,7 +21,7 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
// ---------------------------------------------------------------------------
-// 1. coop_mat_init — float scalar.
+// coop_mat_init — float scalar.
// Expected IR: call __spirv_CompositeConstruct with the scalar value,
// returning a spirv.CooperativeMatrixKHR TargetExtType.
// ---------------------------------------------------------------------------
@@ -37,7 +37,7 @@ kernel void test_init_float(global float *out) {
}
// ---------------------------------------------------------------------------
-// 2. coop_mat_init — integer scalar.
+// coop_mat_init — integer scalar.
// Expected IR: call __spirv_CompositeConstruct with an i32 value.
// ---------------------------------------------------------------------------
@@ -52,7 +52,7 @@ kernel void test_init_int(void) {
}
// ---------------------------------------------------------------------------
-// 3. coop_mat_init — zero initialisation.
+// coop_mat_init — zero initialisation.
// Expected IR: call __spirv_CompositeConstruct with 0.0.
// ---------------------------------------------------------------------------
@@ -67,7 +67,7 @@ kernel void test_init_zero(void) {
}
// ---------------------------------------------------------------------------
-// 4. coop_mat_length — returns unsigned int.
+// coop_mat_length — returns unsigned int.
// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
// an i32 alloca.
// ---------------------------------------------------------------------------
@@ -84,7 +84,7 @@ kernel void test_length(global unsigned int *out) {
}
// ---------------------------------------------------------------------------
-// 5. coop_mat_init then coop_mat_length — combined flow.
+// coop_mat_init then coop_mat_length — combined flow.
// Verifies the TargetExtType produced by init flows into length unchanged.
// ---------------------------------------------------------------------------
@@ -100,7 +100,7 @@ kernel void test_init_then_length(global unsigned int *out) {
}
// ---------------------------------------------------------------------------
-// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// coop_mat_length — calling convention is SPIR_FUNC.
// The call must NOT be a plain 'call' — it must be 'call spir_func'.
// ---------------------------------------------------------------------------
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 10fd9c08173b5..6d4d2bf43b4ea 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,13 +1,6 @@
-// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+// cl_khr_cooperative_matrix registration test.
//
-// Patch 2: cl_khr_cooperative_matrix registration in
-// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
-//
-// Tests: extension macro is predefined when enabled, extension can be
-// explicitly enabled/disabled via pragma, all four enum types and
-// their constants are visible under -finclude-default-header.
-
-// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// Test if extension macro is predefined when the extension is enabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
@@ -15,7 +8,7 @@
// EXT: cl_khr_cooperative_matrix
-// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// Test if extension is NOT predefined when explicitly disabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
@@ -23,7 +16,7 @@
// NOEXT-NOT: cl_khr_cooperative_matrix
-// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// Enum constants are visible when extension is enabled
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -47,7 +40,7 @@ void test_enum_constants(void) {
(void)op;
}
-// ── 2d. Enum constant values match the spec ──────────────────────────────────
+// Enum constant values match the spec
void test_enum_values(void) {
_Static_assert(memory_scope_sub_group == 4, "scope subgroup");
_Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index f16682934759b..4c0763a4d703c 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -1,27 +1,10 @@
-// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
-//
-// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
-// ExprConstant, ItaniumMangle, MicrosoftMangle,
-// ASTReader/ASTWriter TypeLoc serialisation.
-//
-// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// Test Itanium mangling
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
-// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
-// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
-// RUN: -finclude-default-header -O0 -include-pch %t.pch \
-// RUN: -ast-dump %t.pch_probe.cl \
-// RUN: | FileCheck %s --check-prefix=PCH
-//
-// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// Structural equivalence (no diagnostics on compatible pair)
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -39,30 +22,15 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
-// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
-// that contains the vendor-extended "coop_mat" marker.
+// Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
void test_mangling(MatA_t a) { (void)a; }
+// MANGLE: @{{.*}}test_mangling{{.*}}coop_mat(
-// MANGLE: @{{.*}}test_mangling{{.*}}(
-
-// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
-// must still carry a coop_mat parameter type.
-// PCH: FunctionDecl {{.*}} pch_probe
-// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
-
-// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+// Structural equivalence — produce no diagnostic.
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
-
void test_structural_equiv(void) {
MatA_t *p = 0;
MatA_alias *q = p; // same canonical type
(void)q;
}
-
-// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
-// integer, float, pointer …). Using __builtin_classify_type on it
-// compiles without error; result is 0 (no_type_class).
-void test_expr_constant(MatA_t a) {
- int cls = __builtin_classify_type(a);
- (void)cls;
-}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index a9ea17abb2730..5dee7d7a4bef5 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -1,16 +1,10 @@
// Tests for coop_mat_init and coop_mat_length — Sema (positive).
//
-// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
-// so arg-count and arg-type validation are not performed by Sema for these two
-// builtins. Only the positive (valid-code) path and the one assignment-target
-// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
-// tested here.
-//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
-// expected-no-diagnostics (sections 1-6 must compile cleanly)
+// expected-no-diagnostics
// ---------------------------------------------------------------------------
// Type definitions shared across all tests.
@@ -31,9 +25,7 @@ typedef int __attribute__((coop_mat(memory_scope_sub_group,
CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
// ---------------------------------------------------------------------------
-// 1. coop_mat_init: initialise each use role with a float scalar.
-// Two-step (declare then assign) is required because the return type is
-// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// coop_mat_init: initialise with a float scalar.
// ---------------------------------------------------------------------------
void test_coop_mat_init_basic(void) {
MatA_t a;
@@ -45,7 +37,7 @@ void test_coop_mat_init_basic(void) {
}
// ---------------------------------------------------------------------------
-// 2. coop_mat_init: integer element type.
+// coop_mat_init: integer element type.
// ---------------------------------------------------------------------------
void test_coop_mat_init_int(void) {
MatA_int_t m;
@@ -53,7 +45,7 @@ void test_coop_mat_init_int(void) {
}
// ---------------------------------------------------------------------------
-// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// coop_mat_init: re-initialise the same variable (chained assigns).
// ---------------------------------------------------------------------------
void test_coop_mat_init_chained(void) {
MatC_t acc;
@@ -62,8 +54,7 @@ void test_coop_mat_init_chained(void) {
}
// ---------------------------------------------------------------------------
-// 4. coop_mat_length: basic usage — returns unsigned int directly.
-// No two-step required for the length call itself.
+// coop_mat_length: basic usage — returns unsigned int directly.
// ---------------------------------------------------------------------------
void test_coop_mat_length_basic(void) {
MatA_t a;
@@ -73,7 +64,7 @@ void test_coop_mat_length_basic(void) {
}
// ---------------------------------------------------------------------------
-// 5. coop_mat_length: all three use roles.
+// coop_mat_length: all three use roles.
// ---------------------------------------------------------------------------
void test_coop_mat_length_roles(void) {
MatA_t a; a = coop_mat_init(0.0f);
@@ -86,7 +77,7 @@ void test_coop_mat_length_roles(void) {
}
// ---------------------------------------------------------------------------
-// 6. coop_mat_length: result used in arithmetic.
+// coop_mat_length: result used in arithmetic.
// ---------------------------------------------------------------------------
void test_coop_mat_length_arith(void) {
MatA_t a;
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index 7bf1f5d2ae184..63214bc07dc54 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -1,9 +1,5 @@
-// clang/test/SemaOpenCL/coop-mat-sema.cl
+// Tests for semantic checking etc.
//
-// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
-// builtin validation, expr handling, TreeTransform.
-//
-// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
@@ -23,9 +19,7 @@ typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
// ---------------------------------------------------------------------------
-// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
-// match the LHS variable type. Declare first, then assign so that
-// Sema's BuiltinReturnsCoopMatrix path fires correctly.
+// load builtin test
// ---------------------------------------------------------------------------
kernel void test_load_store(__global float *ptr,
__global float *out_ptr) {
@@ -35,7 +29,7 @@ kernel void test_load_store(__global float *ptr,
}
// ---------------------------------------------------------------------------
-// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// mulAdd builtin test
// ---------------------------------------------------------------------------
kernel void test_muladd(__global float *ptr) {
MatA_t a;
@@ -50,7 +44,7 @@ kernel void test_muladd(__global float *ptr) {
}
// ---------------------------------------------------------------------------
-// 4d. Binary element-wise operators (+, -)
+// Binary element-wise operators (+, -) test
// ---------------------------------------------------------------------------
void test_binary_ops(MatA_t a, MatA_t b) {
MatA_t r_add = a + b;
@@ -59,7 +53,7 @@ void test_binary_ops(MatA_t a, MatA_t b) {
}
// ---------------------------------------------------------------------------
-// 4e. Scalar multiply operator
+// Scalar multiply operator test
// ---------------------------------------------------------------------------
void test_scalar_ops(MatA_t a, float s) {
MatA_t r = a * s;
@@ -67,7 +61,7 @@ void test_scalar_ops(MatA_t a, float s) {
}
// ---------------------------------------------------------------------------
-// 4f. Unary minus
+// Unary minus test
// ---------------------------------------------------------------------------
void test_unary_minus(MatA_t a) {
MatA_t r = -a;
@@ -75,8 +69,7 @@ void test_unary_minus(MatA_t a) {
}
// ---------------------------------------------------------------------------
-// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
-// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// Assignment test.
// ---------------------------------------------------------------------------
kernel void test_assignment(__global float *ptr) {
MatA_t a;
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 7f7daab56ebcf..a519754346197 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -1,11 +1,4 @@
-// clang/test/SemaOpenCL/coop_mat_type_infra.cl
-//
-// Patch 1: CooperativeMatrixType AST node and type infrastructure.
-//
-// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
-// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
-// RecursiveASTVisitor traversal, mergeTypes compatibility,
-// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+// CooperativeMatrixType AST node and type infrastructure tests
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
@@ -21,9 +14,6 @@
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
-// ---------------------------------------------------------------------------
-// Enum constants (from opencl-c-base.h via -finclude-default-header)
-// ---------------------------------------------------------------------------
#define SCOPE memory_scope_sub_group
#define USE_A CLK_COOPERATIVE_MATRIX_A
#define USE_B CLK_COOPERATIVE_MATRIX_B
More information about the cfe-commits
mailing list