[Lldb-commits] [clang] [lldb] [llvm] [OpenCL][Clang] Add support for cooperative matrix extension (PR #213986)
via lldb-commits
lldb-commits at lists.llvm.org
Sat Aug 8 21:59:29 PDT 2026
https://github.com/asudarsa-qti updated https://github.com/llvm/llvm-project/pull/213986
>From c064514c9ccf73cac56b4a289e1ad875e3cefe06 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:14:42 -0400
Subject: [PATCH 01/12] [OpenCL][CoopMat] Patch 1 - Introduce Cooperative
matrix type
---
clang/include/clang/AST/ASTContext.h | 10 +++
clang/include/clang/AST/RecursiveASTVisitor.h | 11 +++
clang/include/clang/AST/TypeBase.h | 90 +++++++++++++++++++
clang/include/clang/AST/TypeLoc.h | 34 ++++++-
clang/include/clang/AST/TypeProperties.td | 13 +++
clang/include/clang/Basic/TypeNodes.td | 1 +
.../clang/Serialization/TypeBitCodes.def | 1 +
clang/lib/AST/ASTContext.cpp | 82 +++++++++++++++++
clang/lib/AST/Type.cpp | 38 ++++++++
clang/lib/AST/TypePrinter.cpp | 16 ++++
10 files changed, 295 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 763039e690dec..d3089a4ea5b77 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -246,6 +246,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
mutable llvm::ContextualFoldingSet<DependentVectorType, ASTContext &>
DependentVectorTypes;
mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
+ mutable llvm::FoldingSet<CooperativeMatrixType> CooperativeMatrixTypes;
mutable llvm::ContextualFoldingSet<DependentSizedMatrixType, ASTContext &>
DependentSizedMatrixTypes;
mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
@@ -1853,6 +1854,15 @@ class ASTContext : public RefCountedBase<ASTContext> {
QualType getConstantMatrixType(QualType ElementType, unsigned NumRows,
unsigned NumColumns) const;
+ /// Return the unique reference to the cooperative matrix type of the
+ /// specified element type and size.
+ ///
+ /// \pre \p ElementType must be a valid matrix element type (see
+ /// MatrixType::isValidElementType).
+ QualType getCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use) const;
+
/// Return the unique reference to the matrix type of the specified element
/// type and size
QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 6913e9b614315..af6373df4f7e1 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -1082,6 +1082,9 @@ DEF_TRAVERSE_TYPE(ExtVectorType, { TRY_TO(TraverseType(T->getElementType())); })
DEF_TRAVERSE_TYPE(ConstantMatrixType,
{ TRY_TO(TraverseType(T->getElementType())); })
+DEF_TRAVERSE_TYPE(CooperativeMatrixType,
+ { TRY_TO(TraverseType(T->getElementType())); })
+
DEF_TRAVERSE_TYPE(DependentSizedMatrixType, {
if (T->getRowExpr())
TRY_TO(TraverseStmt(T->getRowExpr()));
@@ -1414,6 +1417,14 @@ DEF_TRAVERSE_TYPELOC(ConstantMatrixType, {
TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
})
+DEF_TRAVERSE_TYPELOC(CooperativeMatrixType, {
+ TRY_TO(TraverseStmt(TL.getAttrScopeOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
+ TRY_TO(TraverseStmt(TL.getAttrUseOperand()));
+ TRY_TO(TraverseType(TL.getTypePtr()->getElementType()));
+})
+
DEF_TRAVERSE_TYPELOC(DependentSizedMatrixType, {
TRY_TO(TraverseStmt(TL.getAttrRowOperand()));
TRY_TO(TraverseStmt(TL.getAttrColumnOperand()));
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 530bfe72dac2b..5ed2039adb816 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -2705,6 +2705,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
bool isSubscriptableVectorType() const;
bool isMatrixType() const; // Matrix type.
bool isConstantMatrixType() const; // Constant matrix type.
+ bool isCooperativeMatrixType() const; // Cooperative matrix type.
bool isOverflowBehaviorType() const; // Overflow behavior type.
bool isDependentAddressSpaceType() const; // value-dependent address space qualifier
bool isObjCObjectPointerType() const; // pointer to ObjC object
@@ -4478,6 +4479,7 @@ class MatrixType : public Type, public llvm::FoldingSetNode {
static bool classof(const Type *T) {
return T->getTypeClass() == ConstantMatrix ||
+ T->getTypeClass() == CooperativeMatrix ||
T->getTypeClass() == DependentSizedMatrix;
}
};
@@ -4567,6 +4569,90 @@ class ConstantMatrixType final : public MatrixType {
}
};
+/// Represents a cooperative matrix type.
+class CooperativeMatrixType final : public MatrixType {
+protected:
+ friend class ASTContext;
+
+ /// Number of rows and columns.
+ unsigned NumRows;
+ unsigned NumColumns;
+
+ /// Scope and use
+ unsigned Scope;
+ unsigned Use;
+
+ static constexpr unsigned MaxElementsPerDimension = (1 << 20) - 1;
+
+ CooperativeMatrixType(QualType MatrixElementType, unsigned Scope,
+ unsigned NRows, unsigned NColumns, unsigned Use,
+ QualType CanonElementType);
+
+ CooperativeMatrixType(TypeClass typeClass, QualType MatrixType,
+ unsigned Scope, unsigned NRows, unsigned NColumns,
+ unsigned Use, QualType CanonElementType);
+
+public:
+ /// Returns the number of rows in the matrix.
+ unsigned getNumRows() const { return NumRows; }
+
+ /// Returns the number of columns in the matrix.
+ unsigned getNumColumns() const { return NumColumns; }
+
+ /// Returns the scope of the matrix.
+ unsigned getScope() const { return Scope; }
+
+ /// Returns the use of the matrix.
+ unsigned getUse() const { return Use; }
+
+ /// Returns the number of elements required to embed the matrix into a vector.
+ unsigned getNumElementsFlattened() const {
+ return getNumRows() * getNumColumns();
+ }
+
+ /// Returns true if \p NumElements is a valid matrix dimension.
+ static constexpr bool isDimensionValid(size_t NumElements) {
+ return NumElements > 0 && NumElements <= MaxElementsPerDimension;
+ }
+
+ /// Return true if \p Scope is valid
+ static constexpr bool isScopeValid(size_t Scope) {
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ }
+
+ /// Return true if \p Use is valid
+ static constexpr bool isUseValid(size_t Use) {
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ }
+
+ /// Returns the maximum number of elements per dimension.
+ static constexpr unsigned getMaxElementsPerDimension() {
+ return MaxElementsPerDimension;
+ }
+
+ void Profile(llvm::FoldingSetNodeID &ID) {
+ Profile(ID, getElementType(), getNumRows(), getNumColumns(), getScope(),
+ getUse(), getTypeClass());
+ }
+
+ static void Profile(llvm::FoldingSetNodeID &ID, QualType ElementType,
+ unsigned NumRows, unsigned NumColumns, unsigned Scope,
+ unsigned Use, TypeClass TypeClass) {
+ ID.AddPointer(ElementType.getAsOpaquePtr());
+ ID.AddInteger(NumRows);
+ ID.AddInteger(NumColumns);
+ ID.AddInteger(Scope);
+ ID.AddInteger(Use);
+ ID.AddInteger(TypeClass);
+ }
+
+ static bool classof(const Type *T) {
+ return T->getTypeClass() == CooperativeMatrix;
+ }
+};
+
/// Represents a matrix type where the type and the number of rows and columns
/// is dependent on a template.
class DependentSizedMatrixType final : public MatrixType {
@@ -8894,6 +8980,10 @@ inline bool Type::isConstantMatrixType() const {
return isa<ConstantMatrixType>(CanonicalType);
}
+inline bool Type::isCooperativeMatrixType() const {
+ return isa<CooperativeMatrixType>(CanonicalType);
+}
+
inline bool Type::isOverflowBehaviorType() const {
return isa<OverflowBehaviorType>(CanonicalType);
}
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 2648806ac687a..731a966d79b69 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -2142,8 +2142,10 @@ class DependentSizedExtVectorTypeLoc
struct MatrixTypeLocInfo {
SourceLocation AttrLoc;
SourceRange OperandParens;
+ Expr *ScopeOperand;
Expr *RowOperand;
Expr *ColumnOperand;
+ Expr *UseOperand;
};
class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
@@ -2151,26 +2153,50 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
public:
/// The location of the attribute name, i.e.
/// float __attribute__((matrix_type(4, 2)))
- /// ^~~~~~~~~~~~~~~~~
+ /// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
SourceLocation getAttrNameLoc() const { return getLocalData()->AttrLoc; }
void setAttrNameLoc(SourceLocation loc) { getLocalData()->AttrLoc = loc; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrScopeOperand() const { return getLocalData()->ScopeOperand; }
+ void setAttrScopeOperand(Expr *e) { getLocalData()->ScopeOperand = e; }
+
/// The attribute's row operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrRowOperand() const { return getLocalData()->RowOperand; }
void setAttrRowOperand(Expr *e) { getLocalData()->RowOperand = e; }
/// The attribute's column operand, if it has one.
/// float __attribute__((matrix_type(4, 2)))
/// ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
Expr *getAttrColumnOperand() const { return getLocalData()->ColumnOperand; }
void setAttrColumnOperand(Expr *e) { getLocalData()->ColumnOperand = e; }
+ /// The attribute's scope operand (only for cooperative matrix).
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^
+ Expr *getAttrUseOperand() const { return getLocalData()->UseOperand; }
+ void setAttrUseOperand(Expr *e) { getLocalData()->UseOperand = e; }
+
/// The location of the parentheses around the operand, if there is
/// an operand.
/// float __attribute__((matrix_type(4, 2)))
/// ^ ^
+ /// For cooperative matrix, it is
+ /// float __attribute__((coop_mat(0, 4, 2, 1)))
+ /// ^ ^ ^
SourceRange getAttrOperandParensRange() const {
return getLocalData()->OperandParens;
}
@@ -2187,8 +2213,10 @@ class MatrixTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, MatrixTypeLoc,
void initializeLocal(ASTContext &Context, SourceLocation loc) {
setAttrNameLoc(loc);
setAttrOperandParensRange(loc);
+ setAttrScopeOperand(nullptr);
setAttrRowOperand(nullptr);
setAttrColumnOperand(nullptr);
+ setAttrUseOperand(nullptr);
}
};
@@ -2196,6 +2224,10 @@ class ConstantMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc, ConstantMatrixTypeLoc,
ConstantMatrixType> {};
+class CooperativeMatrixTypeLoc
+ : public InheritingConcreteTypeLoc<MatrixTypeLoc, CooperativeMatrixTypeLoc,
+ CooperativeMatrixType> {};
+
class DependentSizedMatrixTypeLoc
: public InheritingConcreteTypeLoc<MatrixTypeLoc,
DependentSizedMatrixTypeLoc,
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 1185a3b1dc670..9e86964f2de0d 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -260,6 +260,19 @@ let Class = ConstantMatrixType in {
}]>;
}
+let Class = CooperativeMatrixType in {
+ def : Property<"Scope", UInt32> { let Read = [{ node->getScope() }]; }
+ def : Property<"numRows", UInt32> { let Read = [{ node->getNumRows() }]; }
+ def : Property<"numColumns", UInt32> {
+ let Read = [{ node->getNumColumns() }];
+ }
+ def : Property<"Use", UInt32> { let Read = [{ node->getUse() }]; }
+
+ def : Creator<[{
+ return ctx.getCooperativeMatrixType(elementType, Scope, numRows, numColumns, Use);
+ }]>;
+}
+
let Class = DependentSizedMatrixType in {
def : Property<"rows", ExprRef> {
let Read = [{ node->getRowExpr() }];
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 700a73f669690..e6f396245070c 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -64,6 +64,7 @@ def DependentVectorType : TypeNode<Type>, AlwaysDependent;
def ExtVectorType : TypeNode<VectorType>;
def MatrixType : TypeNode<Type, 1>;
def ConstantMatrixType : TypeNode<MatrixType>;
+def CooperativeMatrixType : TypeNode<MatrixType>;
def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
def FunctionType : TypeNode<Type, 1>;
def FunctionProtoType : TypeNode<FunctionType>;
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 9f1da65e0f940..c9ebac63390f5 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -70,5 +70,6 @@ TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
TYPE_BIT_CODE(OverflowBehavior, OVERFLOWBEHAVIOR, 63)
+TYPE_BIT_CODE(CooperativeMatrix, COOPERATIVE_MATRIX, 64)
#undef TYPE_BIT_CODE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 02a3f88431f58..a04a63a92394c 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2201,6 +2201,17 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
break;
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(T);
+ TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
+ // The internal layout of a matrix value is implementation defined.
+ // Initially be ABI compatible with arrays with respect to alignment and
+ // size.
+ Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
+ Align = ElementInfo.Align;
+ break;
+ }
+
case Type::Builtin:
switch (cast<BuiltinType>(T)->getKind()) {
default: llvm_unreachable("Unknown builtin type!");
@@ -3511,6 +3522,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
case Type::Pipe:
case Type::BitInt:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
OS << "?";
return;
@@ -4342,6 +4354,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
case Type::ExtVector:
case Type::DependentSizedExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::ObjCObject:
@@ -4893,6 +4906,45 @@ QualType ASTContext::getConstantMatrixType(QualType ElementTy, unsigned NumRows,
return QualType(New, 0);
}
+QualType ASTContext::getCooperativeMatrixType(QualType ElementTy,
+ unsigned Scope, unsigned NumRows,
+ unsigned NumColumns,
+ unsigned Use) const {
+ llvm::FoldingSetNodeID ID;
+ CooperativeMatrixType::Profile(ID, ElementTy, Scope, NumRows, NumColumns, Use,
+ Type::CooperativeMatrix);
+
+ assert(MatrixType::isValidElementType(ElementTy, getLangOpts()) &&
+ "need a valid element type");
+ assert(CooperativeMatrixType::isDimensionValid(NumRows) &&
+ CooperativeMatrixType::isDimensionValid(NumColumns) &&
+ "need valid matrix dimensions");
+ assert(CooperativeMatrixType::isScopeValid(Scope) &&
+ "need valid matrix scope");
+ assert(CooperativeMatrixType::isUseValid(Use) && "need valid matrix use");
+ void *InsertPos = nullptr;
+ if (CooperativeMatrixType *MTP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
+ return QualType(MTP, 0);
+
+ QualType Canonical;
+ if (!ElementTy.isCanonical()) {
+ Canonical = getCooperativeMatrixType(getCanonicalType(ElementTy), Scope,
+ NumRows, NumColumns, Use);
+
+ CooperativeMatrixType *NewIP =
+ CooperativeMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
+ assert(!NewIP && "Matrix type shouldn't already exist in the map");
+ (void)NewIP;
+ }
+
+ auto *New = new (*this, TypeAlignment) CooperativeMatrixType(
+ ElementTy, Scope, NumRows, NumColumns, Use, Canonical);
+ CooperativeMatrixTypes.InsertNode(New, InsertPos);
+ Types.push_back(New);
+ return QualType(New, 0);
+}
+
QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
Expr *RowExpr,
Expr *ColumnExpr,
@@ -9769,6 +9821,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
return;
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
if (NotEncodedT)
*NotEncodedT = T;
return;
@@ -10718,6 +10771,18 @@ static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
LHS->getNumColumns() == RHS->getNumColumns();
}
+/// areCompatMatrixTypes - Return true if the two specified matrix types are
+/// compatible.
+static bool areCompatMatrixTypes(const CooperativeMatrixType *LHS,
+ const CooperativeMatrixType *RHS) {
+ assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
+ return LHS->getElementType() == RHS->getElementType() &&
+ LHS->getScope() == RHS->getScope() &&
+ LHS->getNumRows() == RHS->getNumRows() &&
+ LHS->getNumColumns() == RHS->getNumColumns() &&
+ LHS->getUse() == RHS->getUse();
+}
+
bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
QualType SecondVec) {
assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
@@ -12232,6 +12297,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
RHSCan->castAs<ConstantMatrixType>()))
return LHS;
return {};
+ case Type::CooperativeMatrix:
+ if (areCompatMatrixTypes(LHSCan->castAs<CooperativeMatrixType>(),
+ RHSCan->castAs<CooperativeMatrixType>()))
+ return LHS;
+ return {};
case Type::ObjCObject: {
// Check if the types are assignment compatible.
// FIXME: This should be type compatibility, e.g. whether
@@ -14594,6 +14664,17 @@ static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
return Ctx.getConstantMatrixType(getCommonElementType(Ctx, MX, MY),
MX->getNumRows(), MX->getNumColumns());
}
+ case Type::CooperativeMatrix: {
+ const auto *MX = cast<CooperativeMatrixType>(X),
+ *MY = cast<CooperativeMatrixType>(Y);
+ assert(MX->getScope() == MY->getScope());
+ assert(MX->getNumRows() == MY->getNumRows());
+ assert(MX->getNumColumns() == MY->getNumColumns());
+ assert(MX->getUse() == MY->getUse());
+ return Ctx.getCooperativeMatrixType(getCommonElementType(Ctx, MX, MY),
+ MX->getScope(), MX->getNumRows(),
+ MX->getNumColumns(), MX->getUse());
+ }
case Type::DependentSizedMatrix: {
const auto *MX = cast<DependentSizedMatrixType>(X),
*MY = cast<DependentSizedMatrixType>(Y);
@@ -14748,6 +14829,7 @@ static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
CANONICAL_TYPE(ConstantArray)
CANONICAL_TYPE(ArrayParameter)
CANONICAL_TYPE(ConstantMatrix)
+ CANONICAL_TYPE(CooperativeMatrix)
CANONICAL_TYPE(Enum)
CANONICAL_TYPE(ExtVector)
CANONICAL_TYPE(FunctionNoProto)
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 42d148715bc40..ffd83415ec261 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -423,6 +423,20 @@ ConstantMatrixType::ConstantMatrixType(TypeClass tc, QualType matrixType,
: MatrixType(tc, matrixType, canonType), NumRows(nRows),
NumColumns(nColumns) {}
+CooperativeMatrixType::CooperativeMatrixType(QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : CooperativeMatrixType(CooperativeMatrix, matrixType, scope, nRows,
+ nColumns, use, canonType) {}
+
+CooperativeMatrixType::CooperativeMatrixType(TypeClass tc, QualType matrixType,
+ unsigned scope, unsigned nRows,
+ unsigned nColumns, unsigned use,
+ QualType canonType)
+ : MatrixType(tc, matrixType, canonType), NumRows(nRows),
+ NumColumns(nColumns), Scope(scope), Use(use) {}
+
DependentSizedMatrixType::DependentSizedMatrixType(QualType ElementType,
QualType CanonicalType,
Expr *RowExpr,
@@ -1190,6 +1204,18 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
T->getNumColumns());
}
+ QualType VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ QualType elementType = recurse(T->getElementType());
+ if (elementType.isNull())
+ return {};
+ if (elementType.getAsOpaquePtr() == T->getElementType().getAsOpaquePtr())
+ return QualType(T, 0);
+
+ return Ctx.getCooperativeMatrixType(elementType, T->getScope(),
+ T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ }
+
QualType VisitOverflowBehaviorType(const OverflowBehaviorType *T) {
QualType UnderlyingType = recurse(T->getUnderlyingType());
if (UnderlyingType.isNull())
@@ -2071,6 +2097,10 @@ class GetContainedDeducedTypeVisitor
return Visit(T->getElementType());
}
+ Type *VisitCooperativeMatrixType(const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+ }
+
Type *VisitFunctionProtoType(const FunctionProtoType *T) {
if (Syntactic && T->hasTrailingReturn())
return const_cast<FunctionProtoType *>(T);
@@ -3150,6 +3180,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
// in HLSL.
if (Ctx.getLangOpts().HLSL && BaseTy->isConstantMatrixType())
return true;
+ if (Ctx.getLangOpts().HLSL && BaseTy->isCooperativeMatrixType())
+ return true;
// -- a reference type; or
if (BaseTy->isReferenceType())
return true;
@@ -4994,6 +5026,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
return Cache::get(cast<VectorType>(T)->getElementType());
case Type::ConstantMatrix:
return Cache::get(cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return Cache::get(cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return Cache::get(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5096,6 +5130,9 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
case Type::ConstantMatrix:
return computeTypeLinkageInfo(
cast<ConstantMatrixType>(T)->getElementType());
+ case Type::CooperativeMatrix:
+ return computeTypeLinkageInfo(
+ cast<CooperativeMatrixType>(T)->getElementType());
case Type::FunctionNoProto:
return computeTypeLinkageInfo(cast<FunctionType>(T)->getReturnType());
case Type::FunctionProto: {
@@ -5296,6 +5333,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::DependentAddressSpace:
case Type::FunctionProto:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 60964d2859790..1f41a2dbfe741 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -279,6 +279,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -917,6 +918,20 @@ void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
printAfter(T->getElementType(), OS);
}
+void TypePrinter::printCooperativeMatrixBefore(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printBefore(T->getElementType(), OS);
+ OS << " __attribute__((coop_mat(";
+ OS << T->getScope() << ", " << T->getNumRows() << ", ";
+ OS << T->getNumColumns() << ", " << T->getUse();
+ OS << ")))";
+}
+
+void TypePrinter::printCooperativeMatrixAfter(const CooperativeMatrixType *T,
+ raw_ostream &OS) {
+ printAfter(T->getElementType(), OS);
+}
+
void TypePrinter::printDependentSizedMatrixBefore(
const DependentSizedMatrixType *T, raw_ostream &OS) {
if (Policy.UseHLSLTypes) {
@@ -2042,6 +2057,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
case attr::OpenCLConstantAddressSpace:
case attr::OpenCLGenericAddressSpace:
case attr::HLSLGroupSharedAddressSpace:
+ case attr::CoopMatrixType:
// FIXME: Update printAttributedBefore to print these once we generate
// AttributedType nodes for them.
break;
>From e5fd702537b503b0e14e80747fc211b1ee7d2994 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:15:43 -0400
Subject: [PATCH 02/12] Patch 2 - Define extension macro
---
.../include/clang/Basic/OpenCLExtensions.def | 1 +
clang/lib/Headers/opencl-c-base.h | 23 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index f8af9b843f95f..89e96dca85a59 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,6 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
+OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 5e24aa60b8d7f..2dc9b1329d9bb 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -762,4 +762,27 @@ CLINKAGE int printf(__constant const char *st, ...)
// Disable any extensions we may have enabled previously.
#pragma OPENCL EXTENSION all : disable
+typedef enum coop_matrix_scope_t {
+ CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP = 3
+} coop_matrix_scope_t;
+
+typedef enum coop_matrix_operands_t {
+ CLK_COOPERATIVE_MATRIX_OPERAND_NONE = 0,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_A_SIGNED = 0x10,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_B_SIGNED = 0x20,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_C_SIGNED = 0x40,
+ CLK_COOPERATIVE_MATRIX_OPERAND_MATRIX_RESULT_SIGNED = 0x80,
+ CLK_COOPERATIVE_MATRIX_OPERAND_SATURATING_ACCUMULATION = 0x100
+} coop_matrix_operands_t;
+
+typedef enum coop_matrix_use_t {
+ CLK_COOPERATIVE_MATRIX_A = 0,
+ CLK_COOPERATIVE_MATRIX_B = 1,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR = 2
+} coop_matrix_use_t;
+
+typedef enum coop_matrix_layout_t {
+ CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR = 0,
+ CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR = 1
+} coop_matrix_layout_t;
#endif //_OPENCL_BASE_H_
>From cdd7e78b569ee631ee2f8436022faf7fb20badb8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:16:39 -0400
Subject: [PATCH 03/12] Patch 3 - Add support for serialization, round-trip and
mangling
---
clang/lib/AST/ASTImporter.cpp | 11 +++++++++
clang/lib/AST/ASTStructuralEquivalence.cpp | 15 +++++++++++++
clang/lib/AST/ExprConstant.cpp | 1 +
clang/lib/AST/ItaniumMangle.cpp | 26 ++++++++++++++++++++++
clang/lib/AST/MicrosoftMangle.cpp | 21 +++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 10 +++++++++
clang/lib/Serialization/ASTWriter.cpp | 12 ++++++++++
7 files changed, 96 insertions(+)
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 3ad71a223903c..cdf4c133e8565 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2098,6 +2098,17 @@ ExpectedType clang::ASTNodeImporter::VisitConstantMatrixType(
*ToElementTypeOrErr, T->getNumRows(), T->getNumColumns());
}
+ExpectedType clang::ASTNodeImporter::VisitCooperativeMatrixType(
+ const clang::CooperativeMatrixType *T) {
+ ExpectedType ToElementTypeOrErr = import(T->getElementType());
+ if (!ToElementTypeOrErr)
+ return ToElementTypeOrErr.takeError();
+
+ return Importer.getToContext().getCooperativeMatrixType(
+ *ToElementTypeOrErr, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+}
+
ExpectedType clang::ASTNodeImporter::VisitDependentAddressSpaceType(
const clang::DependentAddressSpaceType *T) {
Error Err = Error::success();
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index d8bbfbe5dac72..b6f2c328b5199 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -1142,6 +1142,21 @@ bool ASTStructuralEquivalence::isEquivalent(
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *Mat1 = cast<CooperativeMatrixType>(T1);
+ const CooperativeMatrixType *Mat2 = cast<CooperativeMatrixType>(T2);
+ // The element types must be structurally equivalent and the number of rows
+ // and columns must match.
+ if (!IsStructurallyEquivalent(Context, Mat1->getElementType(),
+ Mat2->getElementType()) ||
+ Mat1->getScope() != Mat2->getScope() ||
+ Mat1->getNumRows() != Mat2->getNumRows() ||
+ Mat1->getNumColumns() != Mat2->getNumColumns() ||
+ Mat1->getUse() != Mat2->getUse())
+ return false;
+ break;
+ }
+
case Type::FunctionProto: {
const auto *Proto1 = cast<FunctionProtoType>(T1);
const auto *Proto2 = cast<FunctionProtoType>(T2);
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 9d69de2a7c6fd..8cddcd07d8bf5 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -16365,6 +16365,7 @@ GCCTypeClass EvaluateBuiltinClassifyType(QualType T,
case Type::BlockPointer:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::ObjCObject:
case Type::ObjCInterface:
case Type::ObjCObjectPointer:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index f8e6b898be250..9f18d3ee8fc4b 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2461,6 +2461,7 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::DependentSizedMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
@@ -4410,6 +4411,31 @@ void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
Out << "E";
}
+void CXXNameMangler::mangleType(const CooperativeMatrixType *T) {
+ // Mangle cooperative matrix types as a vendor extended type:
+ // u<Len>coop_matI<Scope><Rows><Columns><Use><element type>E
+
+ mangleVendorType("coop_mat");
+
+ Out << "I";
+ auto &ASTCtx = getASTContext();
+ unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
+ llvm::APSInt Scope(BitWidth);
+ Scope = T->getScope();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Scope);
+ llvm::APSInt Rows(BitWidth);
+ Rows = T->getNumRows();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
+ llvm::APSInt Columns(BitWidth);
+ Columns = T->getNumColumns();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
+ llvm::APSInt Use(BitWidth);
+ Use = T->getUse();
+ mangleIntegerLiteral(ASTCtx.getSizeType(), Use);
+ mangleType(T->getElementType());
+ Out << "E";
+}
+
void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
// Mangle matrix types as a vendor extended type:
// u<Len>matrix_typeI<row expr><column expr><element type>E
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index a1f2b671d6216..0c54fd41856d1 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3770,6 +3770,27 @@ void MicrosoftCXXNameMangler::mangleType(const ConstantMatrixType *T,
mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
}
+void MicrosoftCXXNameMangler::mangleType(const CooperativeMatrixType *T,
+ Qualifiers quals, SourceRange Range) {
+ QualType EltTy = T->getElementType();
+
+ llvm::SmallString<64> TemplateMangling;
+ llvm::raw_svector_ostream Stream(TemplateMangling);
+ MicrosoftCXXNameMangler Extra(Context, Stream);
+
+ Stream << "?$";
+
+ Extra.mangleSourceName("__coop_mat");
+ Extra.mangleType(EltTy, Range, QMM_Escape);
+
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getScope()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumRows()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumColumns()));
+ Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getUse()));
+
+ mangleArtificialTagType(TagTypeKind::Struct, TemplateMangling, {"__clang"});
+}
+
void MicrosoftCXXNameMangler::mangleType(const DependentSizedMatrixType *T,
Qualifiers quals, SourceRange Range) {
Error(Range.getBegin(), "dependent-sized matrix type") << Range;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 0cb44da23d855..929a895a3dbeb 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7583,6 +7583,16 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ TL.setAttrNameLoc(readSourceLocation());
+ TL.setAttrOperandParensRange(readSourceRange());
+ TL.setAttrScopeOperand(Reader.readExpr());
+ TL.setAttrRowOperand(Reader.readExpr());
+ TL.setAttrColumnOperand(Reader.readExpr());
+ TL.setAttrUseOperand(Reader.readExpr());
+}
+
void TypeLocReader::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index f34b2ff182bc8..794bba58afd06 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -452,6 +452,18 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
+ CooperativeMatrixTypeLoc TL) {
+ addSourceLocation(TL.getAttrNameLoc());
+ SourceRange range = TL.getAttrOperandParensRange();
+ addSourceLocation(range.getBegin());
+ addSourceLocation(range.getEnd());
+ Record.AddStmt(TL.getAttrScopeOperand());
+ Record.AddStmt(TL.getAttrRowOperand());
+ Record.AddStmt(TL.getAttrColumnOperand());
+ Record.AddStmt(TL.getAttrUseOperand());
+}
+
void TypeLocWriter::VisitDependentSizedMatrixTypeLoc(
DependentSizedMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
>From 593cab9ad83cd118f0474fd7ca9f0383338b63fe Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:21 -0400
Subject: [PATCH 04/12] Patch 4 - Add support for semantic analysis and
checking
---
clang/include/clang/Basic/Attr.td | 8 +
.../clang/Basic/DiagnosticSemaKinds.td | 54 +++
clang/include/clang/Sema/Sema.h | 34 +-
clang/lib/Sema/SemaChecking.cpp | 310 +++++++++++++++++-
clang/lib/Sema/SemaDecl.cpp | 32 ++
clang/lib/Sema/SemaExpr.cpp | 131 +++++++-
clang/lib/Sema/SemaLookup.cpp | 1 +
clang/lib/Sema/SemaTemplate.cpp | 5 +
clang/lib/Sema/SemaTemplateDeduction.cpp | 31 ++
clang/lib/Sema/SemaType.cpp | 78 ++++-
clang/lib/Sema/TreeTransform.h | 42 +++
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 39c672322d515..ca3061284946e 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3893,6 +3893,14 @@ def MatrixType : TypeAttr {
let PragmaAttributeSupport = 0;
}
+def CoopMatrixType : TypeAttr {
+ let Spellings = [Clang<"coop_mat">];
+ let Subjects = SubjectList<[TypedefName, Var, ParmVar, Field]>;
+ let Args = [UnsignedArgument<"CoopMatScope">, UnsignedArgument<"NumRows">,
+ UnsignedArgument<"NumColumns">, UnsignedArgument<"CoopMatUse">];
+ let Documentation = [Undocumented];
+}
+
def Visibility : InheritableAttr {
let Clone = 0;
let Spellings = [GCC<"visibility">];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 01527e87c903f..09c789d2d854a 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11982,6 +11982,60 @@ def ext_opencl_ext_vector_type_rgba_selector: ExtWarn<
def err_openclcxx_placement_new : Error<
"use of placement new requires explicit declaration">;
+// OpenCL C Cooperative Matrix.
+def err_invalid_coopmat_attr
+ : Error<"invalid argument of cooperative matrix attribute">;
+
+def err_coop_mat_not_primitive
+ : Error<"cooperative matrix attribute only supported for integer and "
+ "floating-point types">;
+
+def err_coop_mat_rol_col_size
+ : Error<"cooperative matrix row and column must have power-of-two size">;
+
+def err_coop_mem_layout_enum
+ : Error<"memory layout for cooperative matrix should be "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR or "
+ "CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR">;
+
+def err_coop_element_and_pointer_type
+ : Error<"inconsistent between cooperative matrix element type and buffer "
+ "pointer type">;
+
+def err_coop_matrix_arg
+ : Error<"argument must be a valid cooperative matrix type">;
+
+def err_coop_matrix_assignment
+ : Error<"builtin return value should be assigned to cooperative matrix "
+ "type variable">;
+
+def err_coop_matrix_useA : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_A">;
+
+def err_coop_matrix_useB : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_B">;
+
+def err_coop_matrix_useACC : Error<"argument of cooperative matrix must be "
+ "CLK_COOPERATIVE_MATRIX_ACCUMULATOR">;
+
+def err_coop_matrix_element_type
+ : Error<"inconsistent cooperative matrix element type">;
+
+def err_coop_matrix_row_or_col_mismatch
+ : Error<"mismatch of cooperative matrix row or column">;
+
+def err_coop_matrix_use_type
+ : Error<"inconsistent cooperative matrix use type">;
+
+def err_unsupported_coopmat_binary_operator
+ : Error<"unsupported cooperative matrix binary operator">;
+
+def err_unsupported_coopmat_scalar_operator
+ : Error<"unsupported cooperative matrix scalar operator">;
+
+def err_ref_coopmat_var
+ : Error<"invalid reference of cooperative matrix variable">;
+
// MIG routine annotations.
def warn_mig_server_routine_does_not_return_kern_return_t : Warning<
"'mig_server_routine' attribute only applies to routines that return a kern_return_t">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 778c1a2f5c427..9ef1bf75878d3 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3066,6 +3066,34 @@ class Sema final : public SemaBase {
BuiltinCountedByRefKind K);
bool BuiltinCountedByRef(CallExpr *TheCall);
+ // Coop matrix handling.
+ void CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc, CallExpr *call);
+ void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc);
+ bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
+ bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult);
+ void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc);
+ ExprResult BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult CreateCoopMatScalarOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
+ Expr *LHSExpr, Expr *RHSExpr);
+ ExprResult BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult);
+
// Matrix builtin handling.
ExprResult BuiltinMatrixTranspose(CallExpr *TheCall, ExprResult CallResult);
ExprResult BuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
@@ -15306,8 +15334,10 @@ class Sema final : public SemaBase {
/// Run the required checks for the extended vector type.
QualType BuildExtVectorType(QualType T, Expr *ArraySize,
SourceLocation AttrLoc);
+
QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
- SourceLocation AttrLoc);
+ SourceLocation AttrLoc, Expr *Scope = nullptr,
+ Expr *Use = nullptr, bool IsCoopMat = false);
QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
Expr *CountExpr,
@@ -15330,6 +15360,8 @@ class Sema final : public SemaBase {
bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
+ bool IsCoopMatrixBuiltin(Expr *RHSExpr);
+
/// Build a function type.
///
/// This routine checks the function type according to C++ rules and
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index ff42d98df965c..ed4ac3721ad7a 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4048,6 +4048,27 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
break;
}
+ case Builtin::BIcoop_mat_load:
+ return BuiltinCoopMatrixLoad(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_store:
+ return BuiltinCoopMatrixStore(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_mulAdd:
+ return BuiltinCoopMatrixMulAdd(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div:
+ return BuiltinCoopMatrixBinaryOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_mul:
+ return BuiltinCoopMatrixScalarOp(TheCall, TheCallResult);
+
+ case Builtin::BIcoop_mat_scalar_neg:
+ return BuiltinCoopMatrixScalarUnaryOp(TheCall, TheCallResult);
+
case Builtin::BI__builtin_matrix_transpose:
return BuiltinMatrixTranspose(TheCall, TheCallResult);
@@ -17288,6 +17309,270 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
return false;
}
+// Check coop_mat_load/store buffer pointer.
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
+ unsigned PtrArgIdx) {
+ bool ArgError = false;
+ Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
+ ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
+ if (PtrConv.isInvalid())
+ return true;
+ PtrExpr = PtrConv.get();
+ TheCall->setArg(PtrArgIdx, PtrExpr);
+
+ auto *PtrTy = PtrExpr->getType()->getAs<PointerType>();
+ QualType ElementTy;
+ if (!PtrTy) {
+ ArgError = true;
+ } else {
+ ElementTy = PtrTy->getPointeeType().getUnqualifiedType();
+ if (!MatrixType::isValidElementType(ElementTy, getLangOpts())) {
+ ArgError = true;
+ }
+ }
+
+ if (ArgError) {
+ Diag(PtrExpr->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+ << PtrArgIdx + 1 << 0 << /* pointer to element ty */ 5 << /* no fp */ 0
+ << PtrExpr->getType();
+ }
+
+ return ArgError;
+}
+
+// Check coop_mat_load/store matrix element has same type with buffer pointer.
+void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
+ auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
+ if (!MTy) {
+ Diag(MatrixLoc, diag::err_coop_matrix_arg);
+ return;
+ }
+
+ assert(isa<PointerType>(BufferType));
+ auto *PTy = BufferType->castAs<PointerType>();
+
+ if (MTy->getElementType().getUnqualifiedType() !=
+ PTy->getPointeeType().getUnqualifiedType())
+ Diag(MatrixLoc, diag::err_coop_element_and_pointer_type);
+}
+
+void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
+
+ FunctionDecl *F = call->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (Fname->isStr("coop_mat_load"))
+ CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
+ MatrixLoc);
+}
+
+// Check coop_mat_load/store layout argument
+bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
+ bool ArgError = false;
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LayoutExpr);
+ if (DR) {
+ const auto *ECDHS = dyn_cast<EnumConstantDecl>(DR->getDecl());
+ if (ECDHS) {
+ if (ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+ ArgError = true;
+ } else
+ ArgError = true;
+ } else
+ ArgError = true;
+
+ if (ArgError)
+ Diag(LayoutExpr->getBeginLoc(), diag::err_coop_mem_layout_enum);
+
+ return ArgError;
+}
+
+ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
+ return ExprError();
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(1)))
+ return ExprError();
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 4))
+ return ExprError();
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
+ return ExprError();
+ CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
+ Arg0->getBeginLoc());
+ if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
+ return ExprError();
+ return CallResult;
+}
+
+void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
+ FunctionDecl *F = TheCall->getDirectCallee();
+ assert(F);
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ assert(Fname);
+ if (!Fname->isStr("coop_mat_mulAdd"))
+ return;
+
+ auto MC = TheCall->getArg(2);
+ auto *MOutTy = TheCall->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = MC->getType()->getAs<CooperativeMatrixType>();
+ auto Loc = TheCall->getBeginLoc();
+
+ if (!MOutTy)
+ Diag(Loc, diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(MC->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!MOutTy || !M2Ty)
+ return;
+
+ if (MOutTy->getUse() != 2)
+ Diag(Loc, diag::err_coop_matrix_useACC);
+
+ if (MOutTy->getElementType().getUnqualifiedType() !=
+ M2Ty->getElementType().getUnqualifiedType())
+ Diag(Loc, diag::err_coop_matrix_element_type);
+
+ if (!areMatrixTypesOfTheSameDimension(TheCall->getType(), MC->getType()))
+ Diag(Loc, diag::err_coop_matrix_row_or_col_mismatch);
+}
+
+bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
+ SourceLocation BLoc) {
+ auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
+ auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
+ if (!M0Ty)
+ Diag(ALoc, diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(BLoc, diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty)
+ return true;
+
+ if (!areMatrixTypesOfTheSameDimension(ATy, BTy)) {
+ Diag(ALoc, diag::err_coop_matrix_row_or_col_mismatch);
+ return true;
+ }
+
+ if (M0Ty->getUse() != M1Ty->getUse()) {
+ Diag(ALoc, diag::err_coop_matrix_use_type);
+ return true;
+ }
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType()) {
+ Diag(ALoc, diag::err_coop_matrix_element_type);
+ return true;
+ }
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+
+ CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
+ Arg1->getBeginLoc());
+
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
+ if (ATy->isIntegerType() && CTy->isIntegerType())
+ return true;
+ if (ATy->isFloatingType() && CTy->isFloatingType())
+ return true;
+ return false;
+}
+
+ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 3))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ Expr *Arg1 = TheCall->getArg(1);
+ Expr *Arg2 = TheCall->getArg(2);
+
+ auto *M0Ty = Arg0->getType()->getAs<CooperativeMatrixType>();
+ auto *M1Ty = Arg1->getType()->getAs<CooperativeMatrixType>();
+ auto *M2Ty = Arg2->getType()->getAs<CooperativeMatrixType>();
+ auto Loc0 = Arg0->getBeginLoc();
+ auto Loc1 = Arg1->getBeginLoc();
+
+ if (!M0Ty)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M1Ty)
+ Diag(Arg1->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M2Ty)
+ Diag(Arg2->getBeginLoc(), diag::err_coop_matrix_arg);
+ if (!M0Ty || !M1Ty || !M2Ty)
+ return ExprError();
+
+ if (M0Ty->getUse() != 0)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useA);
+ if (M1Ty->getUse() != 1)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useB);
+ if (M2Ty->getUse() != 2)
+ Diag(Arg0->getBeginLoc(), diag::err_coop_matrix_useACC);
+
+ if (M0Ty->getElementType().getUnqualifiedType() !=
+ M1Ty->getElementType().getUnqualifiedType())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_element_type));
+
+ if (!isValidMatAMatCElementTypeCombination(M0Ty->getElementType(),
+ M2Ty->getElementType()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_element_type));
+
+ if (M0Ty->getNumRows() != M2Ty->getNumRows())
+ return ExprError(Diag(Loc0, diag::err_coop_matrix_row_or_col_mismatch));
+ if ((M1Ty->getNumColumns() != M2Ty->getNumColumns()) ||
+ (M0Ty->getNumColumns() != M1Ty->getNumRows()))
+ return ExprError(Diag(Loc1, diag::err_coop_matrix_row_or_col_mismatch));
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 2))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
+ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
+ ExprResult CallResult) {
+ if (checkArgCount(TheCall, 1))
+ return ExprError();
+
+ Expr *Arg0 = TheCall->getArg(0);
+ TheCall->setType(Arg0->getType());
+
+ return CallResult;
+}
+
ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
@@ -17298,8 +17583,9 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
return MatrixArg;
Expr *Matrix = MatrixArg.get();
- auto *MType = Matrix->getType()->getAs<ConstantMatrixType>();
- if (!MType) {
+ auto *ConstMType = Matrix->getType()->getAs<ConstantMatrixType>();
+ auto *CoopMType = Matrix->getType()->getAs<CooperativeMatrixType>();
+ if (!ConstMType && !CoopMType) {
Diag(Matrix->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* matrix */ 3 << /* no int */ 0 << /* no fp */ 0
<< Matrix->getType();
@@ -17308,11 +17594,23 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// Create returned matrix type by swapping rows and columns of the argument
// matrix type.
- QualType ResultType = Context.getConstantMatrixType(
- MType->getElementType(), MType->getNumColumns(), MType->getNumRows());
+ if (ConstMType) {
+ QualType ResultType = Context.getConstantMatrixType(
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
- // Change the return type to the type of the returned matrix.
- TheCall->setType(ResultType);
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
+ if (CoopMType) {
+ QualType ResultType = Context.getCooperativeMatrixType(
+ CoopMType->getElementType(),CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
+
+ // Change the return type to the type of the returned matrix.
+ TheCall->setType(ResultType);
+ }
// Update call argument to use the possibly converted matrix argument.
TheCall->setArg(0, Matrix);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a553c6994f0ed..8e1e217b258c7 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -13946,6 +13946,25 @@ void Sema::DiagnoseUniqueObjectDuplication(const VarDecl *VD) {
}
}
+// Return true if RHSExpr is a cooperative matrix builtin call.
+bool Sema::IsCoopMatrixBuiltin(Expr *RHSExpr) {
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ if (!call)
+ return false;
+ FunctionDecl *F = call->getDirectCallee();
+ if (!F)
+ return false;
+ DeclarationName MemberName = F->getDeclName();
+ IdentifierInfo *Fname = MemberName.getAsIdentifierInfo();
+ if (!Fname)
+ return false;
+ if (Fname->getName().starts_with("coop_mat") &&
+ !Fname->getName().starts_with("coop_mat_length"))
+ return true;
+
+ return false;
+}
+
void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
llvm::scope_exit ResetDeclForInitializer([this]() {
if (!this->ExprEvalContexts.empty())
@@ -14464,6 +14483,19 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
if (LangOpts.OpenACC && !InitType.isNull())
OpenACC().ActOnVariableInit(VDecl, InitType);
+
+ // Set return type of builtin call using type of LHS variable.
+ // This is done for builtin calls that return cooperative matrix.
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(Init)) {
+ if (!VDecl->getType()->isMatrixType()) {
+ Diag(VDecl->getLocation(), diag::err_coop_matrix_assignment);
+ return;
+ }
+
+ auto call = dyn_cast<CallExpr>(Init);
+ assert(call);
+ call->setType(VDecl->getType());
+ }
}
void Sema::ActOnInitializerError(Decl *D) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index ed3d27b5adc27..333212ef369a1 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4626,6 +4626,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::TemplateSpecialization:
@@ -7941,6 +7942,16 @@ bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
if (!destTy->isMatrixType() || !srcTy->isMatrixType())
return false;
+ if (srcTy->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *matSrcType =
+ srcTy->getAs<CooperativeMatrixType>();
+ const CooperativeMatrixType *matDestType =
+ destTy->getAs<CooperativeMatrixType>();
+
+ return matSrcType->getNumRows() == matDestType->getNumRows() &&
+ matSrcType->getNumColumns() == matDestType->getNumColumns();
+ }
+
const ConstantMatrixType *matSrcType = srcTy->getAs<ConstantMatrixType>();
const ConstantMatrixType *matDestType = destTy->getAs<ConstantMatrixType>();
@@ -11284,12 +11295,14 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
if (!IsDiv &&
- (LHSTy->isConstantMatrixType() || RHSTy->isConstantMatrixType()))
+ (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
if (IsDiv && LHSTy->isConstantMatrixType() && RHSTy->isArithmeticType())
return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
+ if (IsDiv && LHSTy->isCooperativeMatrixType() && RHSTy->isArithmeticType())
+ return CheckMatrixElementwiseOperands(LHS, RHS, Loc, IsCompAssign);
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
@@ -11691,6 +11704,15 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -11818,6 +11840,15 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
return compType;
}
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ QualType compType =
+ CheckMatrixElementwiseOperands(LHS, RHS, Loc, CompLHSTy);
+ if (CompLHSTy)
+ *CompLHSTy = compType;
+ return compType;
+ }
+
QualType compType = UsualArithmeticConversions(
LHS, RHS, Loc,
CompLHSTy ? ArithConvKind::CompAssign : ArithConvKind::Arithmetic);
@@ -13812,6 +13843,32 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
if (RHS.isInvalid())
return QualType();
+ if (LHS.get()->getType()->isCooperativeMatrixType() ||
+ RHS.get()->getType()->isCooperativeMatrixType()) {
+ auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
+ auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
+ assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ if (LHSMatType && RHSMatType) {
+ if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
+ return InvalidOperands(Loc, LHS, RHS);
+
+ if (Context.hasSameType(LHSMatType, RHSMatType))
+ return Context.getCommonSugaredType(
+ LHS.get()->getType().getUnqualifiedType(),
+ RHS.get()->getType().getUnqualifiedType());
+
+ QualType LHSELTy = LHSMatType->getElementType(),
+ RHSELTy = RHSMatType->getElementType();
+ if (!Context.hasSameType(LHSELTy, RHSELTy))
+ return InvalidOperands(Loc, LHS, RHS);
+
+ return Context.getCooperativeMatrixType(
+ Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
+ LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
+ LHSMatType->getUse());
+ }
+ }
+
auto *LHSMatType = LHS.get()->getType()->getAs<ConstantMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<ConstantMatrixType>();
assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
@@ -15524,6 +15581,47 @@ static bool needsConversionOfHalfVec(bool OpRequiresConversion, ASTContext &Ctx,
return HasVectorOfHalfType(E0) && (!E1 || HasVectorOfHalfType(E1));
}
+ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Add:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
+ Args);
+ case BO_Sub:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
+ Args);
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
+ Args);
+ case BO_Div:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_binary_operator));
+}
+
+ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
+ BinaryOperatorKind Opc, Expr *LHSExpr,
+ Expr *RHSExpr) {
+ SmallVector<Expr *, 2> Args;
+ Args.push_back(LHSExpr);
+ Args.push_back(RHSExpr);
+ switch (Opc) {
+ case BO_Mul:
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
+ Args);
+ default:
+ break;
+ }
+ return ExprError(Diag(OpLoc, diag::err_unsupported_coopmat_scalar_operator));
+}
+
ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
BinaryOperatorKind Opc, Expr *LHSExpr,
Expr *RHSExpr, bool ForFoldExpression) {
@@ -15560,6 +15658,17 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
if (getLangOpts().OpenCL) {
QualType LHSTy = LHSExpr->getType();
QualType RHSTy = RHSExpr->getType();
+ // Cooperative matrix support
+ if (LHSTy->isMatrixType() && RHSTy->isMatrixType()) {
+ // Check matrix types for assignment.
+ if (BO_Assign == Opc) {
+ if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
+ RHSExpr->getBeginLoc()))
+ return ExprError();
+ } else
+ return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
+ } else if (LHSTy->isMatrixType() && RHSTy->isScalarType())
+ return CreateCoopMatScalarOp(OpLoc, Opc, LHSExpr, RHSExpr);
// OpenCLC v2.0 s6.13.11.1 allows atomic variables to be initialized by
// the ATOMIC_VAR_INIT macro.
if (LHSTy->isAtomicType() || RHSTy->isAtomicType()) {
@@ -15587,6 +15696,18 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
switch (Opc) {
case BO_Assign:
+ if (getLangOpts().OpenCL && IsCoopMatrixBuiltin(RHSExpr)) {
+ if (!LHSExpr->getType()->isMatrixType()) {
+ Diag(LHSExpr->getBeginLoc(), diag::err_coop_matrix_assignment);
+ return ExprError();
+ }
+ auto call = dyn_cast<CallExpr>(RHSExpr);
+ assert(call);
+ call->setType(LHSExpr->getType());
+ CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
+ call);
+ CheckCoopMatrixMatMulOutput(call);
+ }
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
if (getLangOpts().CPlusPlus &&
LHS.get()->getObjectKind() != OK_ObjCProperty) {
@@ -16246,6 +16367,10 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
bool ConvertHalfVec = false;
if (getLangOpts().OpenCL) {
QualType Ty = InputExpr->getType();
+ if (Opc == UO_Minus && Ty->isMatrixType()) {
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_neg,
+ {InputExpr});
+ }
// The only legal unary operation for atomics is '&'.
if ((Opc != UO_AddrOf && Ty->isAtomicType()) ||
// OpenCL special types - image, sampler, pipe, and blocks are to be used
@@ -16323,7 +16448,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
Opc == UO_Plus && resultType->isPointerType())
break;
-
+ else if (getLangOpts().OpenCL && Opc == UO_Minus &&
+ resultType->isCooperativeMatrixType())
+ break;
return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
<< resultType << Input.get()->getSourceRange());
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 43129800e9813..394487561dd92 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3265,6 +3265,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BitInt:
break;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 643392833759d..5ac0492cb4166 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6368,6 +6368,11 @@ bool UnnamedLocalNoLinkageFinder::VisitConstantMatrixType(
return Visit(T->getElementType());
}
+bool UnnamedLocalNoLinkageFinder::VisitCooperativeMatrixType(
+ const CooperativeMatrixType *T) {
+ return Visit(T->getElementType());
+}
+
bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
const FunctionProtoType* T) {
for (const auto &A : T->param_types()) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 3c45806c47a6e..de86af54e69d8 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2445,6 +2445,30 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
&DependentSizedMatrixType::getColumnExpr);
}
+ // (clang extension)
+ //
+ // T __attribute__((coop_mat_type(<integral constant>,
+ // <integral constant>,
+ // <integral constant>,
+ // <integral constant>)))
+ case Type::CooperativeMatrix: {
+ const auto *MP = P->castAs<CooperativeMatrixType>(),
+ *MA = A->getAs<CooperativeMatrixType>();
+ if (!MA)
+ return TemplateDeductionResult::NonDeducedMismatch;
+
+ // Check that the dimensions are the same
+ if (MP->getNumRows() != MA->getNumRows() ||
+ MP->getNumColumns() != MA->getNumColumns()) {
+ return TemplateDeductionResult::NonDeducedMismatch;
+ }
+ // Perform deduction on element types.
+ return DeduceTemplateArgumentsByTypeMatch(
+ S, TemplateParams, MP->getElementType(), MA->getElementType(), Info,
+ Deduced, TDF, degradeCallPartialOrderingKind(POK),
+ /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+ }
+
// (clang extension)
//
// T __attribute__(((address_space(N))))
@@ -6925,6 +6949,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *MatType = cast<CooperativeMatrixType>(T);
+ MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
+ Depth, Used);
+ break;
+ }
+
case Type::DependentSizedMatrix: {
const DependentSizedMatrixType *MatType = cast<DependentSizedMatrixType>(T);
MarkUsedTemplateParameters(Ctx, MatType->getElementType(), OnlyDeduced,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index dc3564c8b17fd..00579d7136825 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -2496,9 +2496,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *SizeExpr,
}
QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
- SourceLocation AttrLoc) {
- assert(Context.getLangOpts().MatrixTypes &&
- "Should never build a matrix type when it is disabled");
+ SourceLocation AttrLoc, Expr *ScopeExpr,
+ Expr *UseExpr, bool IsCoopMat) {
+ if (!IsCoopMat)
+ assert(Context.getLangOpts().MatrixTypes &&
+ "Should never build a matrix type when it is disabled");
// Check element type, if it is not dependent.
if (!ElementTy->isDependentType() &&
@@ -2579,6 +2581,26 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
<< ColRange << "matrix column";
return QualType();
}
+ if (IsCoopMat) {
+ std::optional<llvm::APSInt> ValueScope =
+ ScopeExpr->getIntegerConstantExpr(Context);
+ unsigned Scope = static_cast<unsigned>(ValueScope->getZExtValue());
+ std::optional<llvm::APSInt> ValueUse =
+ UseExpr->getIntegerConstantExpr(Context);
+ unsigned Use = static_cast<unsigned>(ValueUse->getZExtValue());
+
+ if (!CooperativeMatrixType::isScopeValid(Scope)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr)
+ << ColRange << "matrix scope";
+ return QualType();
+ }
+ if (!CooperativeMatrixType::isUseValid(Use)) {
+ Diag(AttrLoc, diag::err_invalid_coopmat_attr) << ColRange << "matrix use";
+ return QualType();
+ }
+ return Context.getCooperativeMatrixType(ElementTy, Scope, MatrixRows,
+ MatrixColumns, Use);
+ }
return Context.getConstantMatrixType(ElementTy, MatrixRows, MatrixColumns);
}
@@ -5944,6 +5966,23 @@ static void fillAtomicQualLoc(AtomicTypeLoc ATL, const DeclaratorChunk &Chunk) {
ATL.setParensRange(SourceRange());
}
+static void fillCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc MTL,
+ const ParsedAttributesView &Attrs) {
+ for (const ParsedAttr &AL : Attrs) {
+ if (AL.getKind() == ParsedAttr::AT_CoopMatrixType) {
+ MTL.setAttrNameLoc(AL.getLoc());
+ MTL.setAttrScopeOperand(AL.getArgAsExpr(0));
+ MTL.setAttrRowOperand(AL.getArgAsExpr(1));
+ MTL.setAttrColumnOperand(AL.getArgAsExpr(2));
+ MTL.setAttrUseOperand(AL.getArgAsExpr(3));
+ MTL.setAttrOperandParensRange(SourceRange());
+ return;
+ }
+ }
+
+ llvm_unreachable("no matrix_type attribute found at the expected location!");
+}
+
namespace {
class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
Sema &SemaRef;
@@ -6337,6 +6376,9 @@ namespace {
void VisitMatrixTypeLoc(MatrixTypeLoc TL) {
fillMatrixTypeLoc(TL, Chunk.getAttrs());
}
+ void VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
+ fillCooperativeMatrixTypeLoc(TL, Chunk.getAttrs());
+ }
void VisitTypeLoc(TypeLoc TL) {
llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -8932,23 +8974,28 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
/// HandleMatrixTypeAttr - "matrix_type" attribute, like ext_vector_type
static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
- Sema &S) {
- if (!S.getLangOpts().MatrixTypes) {
+ Sema &S, bool IsCoopMat = false) {
+ if (!S.getLangOpts().MatrixTypes && !IsCoopMat) {
S.Diag(Attr.getLoc(), diag::err_builtin_matrix_disabled);
return;
}
- if (Attr.getNumArgs() != 2) {
+ unsigned int NumAttrs = IsCoopMat ? 4 : 2;
+ if (Attr.getNumArgs() != NumAttrs) {
S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
- << Attr << 2;
+ << Attr << NumAttrs;
return;
}
-
- Expr *RowsExpr = Attr.getArgAsExpr(0);
- Expr *ColsExpr = Attr.getArgAsExpr(1);
- QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
- if (!T.isNull())
- CurType = T;
+ if (IsCoopMat) {
+ Expr *Scope = Attr.getArgAsExpr(0);
+ Expr *RowsExpr = Attr.getArgAsExpr(1);
+ Expr *ColsExpr = Attr.getArgAsExpr(2);
+ Expr *Use = Attr.getArgAsExpr(3);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc(),
+ Scope, Use, /* IsCoopMat */ true);
+ if (!T.isNull())
+ CurType = T;
+ }
}
static void HandleAnnotateTypeAttr(TypeProcessingState &State,
@@ -9220,6 +9267,11 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
attr.setUsedAsTypeAttr();
break;
+ case ParsedAttr::AT_CoopMatrixType:
+ HandleMatrixTypeAttr(type, attr, state.getSema(), true /* IsCoopMat */);
+ attr.setUsedAsTypeAttr();
+ break;
+
case ParsedAttr::AT_WebAssemblyFuncref: {
if (!HandleWebAssemblyFuncrefAttr(state, type, attr))
attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2083fcd372e81..2e7b652c244f6 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1044,6 +1044,12 @@ class TreeTransform {
QualType RebuildConstantMatrixType(QualType ElementType, unsigned NumRows,
unsigned NumColumns);
+ /// Build a new cooperative matrix type given the element type and
+ /// scope and use and dimensions.
+ QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
+ unsigned NumRows, unsigned NumColumns,
+ unsigned Use);
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
@@ -6180,6 +6186,34 @@ TreeTransform<Derived>::TransformConstantMatrixType(TypeLocBuilder &TLB,
return Result;
}
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformCooperativeMatrixType(
+ TypeLocBuilder &TLB, CooperativeMatrixTypeLoc TL) {
+ const CooperativeMatrixType *T = TL.getTypePtr();
+ QualType ElementType = getDerived().TransformType(T->getElementType());
+ if (ElementType.isNull())
+ return QualType();
+
+ QualType Result = TL.getType();
+ if (getDerived().AlwaysRebuild() || ElementType != T->getElementType()) {
+ Result = getDerived().RebuildCooperativeMatrixType(
+ ElementType, T->getScope(), T->getNumRows(), T->getNumColumns(),
+ T->getUse());
+ if (Result.isNull())
+ return QualType();
+ }
+
+ CooperativeMatrixTypeLoc NewTL = TLB.push<CooperativeMatrixTypeLoc>(Result);
+ NewTL.setAttrNameLoc(TL.getAttrNameLoc());
+ NewTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+ NewTL.setAttrScopeOperand(TL.getAttrScopeOperand());
+ NewTL.setAttrRowOperand(TL.getAttrRowOperand());
+ NewTL.setAttrColumnOperand(TL.getAttrColumnOperand());
+ NewTL.setAttrUseOperand(TL.getAttrUseOperand());
+
+ return Result;
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::TransformDependentSizedMatrixType(
TypeLocBuilder &TLB, DependentSizedMatrixTypeLoc TL) {
@@ -17979,6 +18013,14 @@ QualType TreeTransform<Derived>::RebuildConstantMatrixType(
NumColumns);
}
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildCooperativeMatrixType(
+ QualType ElementType, unsigned Scope, unsigned NumRows, unsigned NumColumns,
+ unsigned Use) {
+ return SemaRef.Context.getCooperativeMatrixType(ElementType, Scope, NumRows,
+ NumColumns, Use);
+}
+
template <typename Derived>
QualType TreeTransform<Derived>::RebuildDependentSizedMatrixType(
QualType ElementType, Expr *RowExpr, Expr *ColumnExpr,
>From 0b0cc80442087e70bc6e42b4144f75c9aaaa2f15 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:17:56 -0400
Subject: [PATCH 05/12] Patch 5 - Emit LLVM IR and intrinsics
---
clang/include/clang/Basic/Builtins.td | 67 +++++++
clang/lib/CodeGen/CGBuiltin.cpp | 275 ++++++++++++++++++++++++++
clang/lib/CodeGen/CGDebugInfo.cpp | 29 +++
clang/lib/CodeGen/CGDebugInfo.h | 1 +
clang/lib/CodeGen/CGExpr.cpp | 61 ++++++
clang/lib/CodeGen/CGExprScalar.cpp | 14 ++
clang/lib/CodeGen/CodeGenFunction.cpp | 2 +
clang/lib/CodeGen/CodeGenFunction.h | 9 +
clang/lib/CodeGen/CodeGenTBAA.cpp | 2 +-
clang/lib/CodeGen/CodeGenTypes.cpp | 33 +++-
clang/lib/CodeGen/ItaniumCXXABI.cpp | 2 +
clang/lib/CodeGen/QualTypeMapper.cpp | 6 +
clang/lib/CodeGen/QualTypeMapper.h | 1 +
13 files changed, 500 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index ea8dbb96fab56..6b19fea681489 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,6 +1931,73 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
+// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+def CoopMatLoad : Builtin {
+ let Spellings = ["coop_mat_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatStore : Builtin {
+ let Spellings = ["coop_mat_store"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatMulAdd : Builtin {
+ let Spellings = ["coop_mat_mulAdd"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryAdd : Builtin {
+ let Spellings = ["coop_mat_binary_add"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinarySub : Builtin {
+ let Spellings = ["coop_mat_binary_sub"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryMul : Builtin {
+ let Spellings = ["coop_mat_binary_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatBinaryDiv : Builtin {
+ let Spellings = ["coop_mat_binary_div"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarMul : Builtin {
+ let Spellings = ["coop_mat_scalar_mul"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatScalarNeg : Builtin {
+ let Spellings = ["coop_mat_scalar_neg"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatInit : Builtin {
+ let Spellings = ["coop_mat_init"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
+def CoopMatLength : Builtin {
+ let Spellings = ["coop_mat_length"];
+ let Attributes = [NoThrow, Const, CustomTypeChecking];
+ let Prototype = "unsigned int(...)";
+}
+
// "Overloaded" Atomic operator builtins. These are overloaded to support data
// types of i8, i16, i32, i64, and i128. The front-end sees calls to the
// non-suffixed version of these (which has a bogus type) and transforms them to
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index f38a891272474..676ee0cc5b4ad 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2942,6 +2942,52 @@ struct PaddingClearer {
llvm::SmallVector<BitInterval> OccuppiedIntervals;
};
+static std::string getSPIRVBuiltinName(unsigned BuiltinID,
+ bool IsFloat = false) {
+ switch (BuiltinID) {
+ case Builtin::BIcoop_mat_load:
+ return "__spirv_CooperativeMatrixLoadKHR";
+ case Builtin::BIcoop_mat_store:
+ return "__spirv_CooperativeMatrixStoreKHR";
+ case Builtin::BIcoop_mat_mulAdd:
+ return "__spirv_CooperativeMatrixMulAddKHR";
+ case Builtin::BIcoop_mat_binary_add:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFAdd" : "__spirv_IAdd";
+ case Builtin::BIcoop_mat_binary_sub:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFSub" : "__spirv_ISub";
+ case Builtin::BIcoop_mat_binary_mul:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFMul" : "__spirv_IMul";
+ case Builtin::BIcoop_mat_binary_div:
+ return (IsFloat) ? "__spirv_CooperativeMatrixFDiv" : "__spirv_IDiv";
+ case Builtin::BIcoop_mat_scalar_mul:
+ return "__spirv_CooperativeMatrixScalarMulKHR";
+ case Builtin::BIcoop_mat_scalar_neg:
+ return "__spirv_CooperativeMatrixScalarNeg";
+ case Builtin::BIcoop_mat_init:
+ return "__spirv_CompositeConstruct";
+ case Builtin::BIcoop_mat_length:
+ return "__spirv_CooperativeMatrixLengthKHR";
+ }
+ assert(0 && "Unexpected Builtin");
+ return "";
+}
+
+static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
+ CodeGenModule &CGM,
+ const CooperativeMatrixType *MTy) {
+ llvm::Type *ElTy = CGF.ConvertType(MTy->getElementType());
+ // Type arguments for TargetExtType
+ llvm::Type *Tys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
+ MTy->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = llvm::TargetExtType::get(
+ CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(Tys), llvm::ArrayRef<unsigned>(Ints));
+ return RetType;
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4799,6 +4845,235 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(Result);
}
+ case Builtin::BIcoop_mat_load: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(SrcAddr, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_load without coop_mat output operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Layout = EmitScalarExpr(E->getArg(1));
+ auto Stride = EmitScalarExpr(E->getArg(2));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *RetType = getTargetExtType(*this, CGM, MTy);
+
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_store: {
+ auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
+ assert(PtrTy && "arg0 must be of pointer type");
+
+ Address Dst = EmitPointerWithAlignment(E->getArg(0));
+ EmitNonNullArgCheck(Dst, E->getArg(0)->getType(),
+ E->getArg(0)->getExprLoc(), FD, 0);
+ const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_store without coop_mat input operand");
+
+ auto Ptr = EmitScalarExpr(E->getArg(0));
+ auto Arg0 = EmitScalarExpr(E->getArg(1));
+ auto Layout = EmitScalarExpr(E->getArg(2));
+ auto Stride = EmitScalarExpr(E->getArg(3));
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+
+ llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_mulAdd: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ if (!MTy)
+ CGM.ErrorUnsupported(
+ E, "coop_mat_mulAdd without coop_mat output operand");
+
+ auto Arg0 = E->getArg(0);
+ auto Arg1 = E->getArg(1);
+ auto Arg2 = E->getArg(2);
+ auto MA = EmitScalarExpr(Arg0);
+ auto MB = EmitScalarExpr(Arg1);
+ auto MC = EmitScalarExpr(Arg2);
+ const auto *MATy = Arg0->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = Arg1->getType()->getAs<CooperativeMatrixType>();
+ const auto *MCTy = Arg2->getType()->getAs<CooperativeMatrixType>();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MATy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ auto *CType = getTargetExtType(*this, CGM, MCTy);
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, CType, Int1Ty}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_binary_add:
+ case Builtin::BIcoop_mat_binary_sub:
+ case Builtin::BIcoop_mat_binary_mul:
+ case Builtin::BIcoop_mat_binary_div: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = getTargetExtType(*this, CGM, MBTy);
+ llvm::Type *ElTy = ConvertType(MTy->getElementType());
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name = (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_mul: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ auto Arg1 = EmitScalarExpr(E->getArg(1));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+ auto *BType = Arg1->getType();
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, BType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_scalar_neg: {
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ auto *AType = getTargetExtType(*this, CGM, MATy);
+
+ // Check if the data is signed/unsigned
+ QualType QT = MTy->getElementType();
+ bool isSigned = false;
+ if (const BuiltinType *BT = QT->getAs<BuiltinType>()) {
+ isSigned = BT->isSignedInteger();
+ }
+ llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
+ llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
+
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {AType, Int1Ty}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_init: {
+ const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
+ auto Init = EmitScalarExpr(E->getArg(0));
+ auto *RetType = getTargetExtType(*this, CGM, MTy);
+ // Set function type.
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ RetType, {Init->getType()}, false);
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(InitFn, {Init});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
+ case Builtin::BIcoop_mat_length: {
+ const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
+ auto *AType = getTargetExtType(*this, CGM, MTy);
+ auto Arg0 = EmitScalarExpr(E->getArg(0));
+ llvm::FunctionType *FTy = llvm::FunctionType::get(
+ ConvertType(E->getType()), {AType}, false);
+
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
+ F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
+ NewCall->setCallingConv(llvm::CallingConv::SPIR_FUNC);
+ return RValue::get(NewCall);
+ }
+
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load: {
llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index f155761a1d998..07848ebdf00e4 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3836,6 +3836,33 @@ llvm::DIType *CGDebugInfo::CreateType(const ConstantMatrixType *Ty,
return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
}
+llvm::DIType *CGDebugInfo::CreateType(const CooperativeMatrixType *Ty,
+ llvm::DIFile *Unit) {
+ // FIXME: Create another debug type for matrices
+ // For the time being, it treats it like a nested ArrayType.
+
+ llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit);
+ uint64_t Size = CGM.getContext().getTypeSize(Ty);
+ uint32_t Align = getTypeAlignIfRequired(Ty, CGM.getContext());
+
+ // Create ranges for both dimensions.
+ llvm::SmallVector<llvm::Metadata *, 2> Subscripts;
+ auto *ColumnCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumColumns()));
+ auto *RowCountNode =
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
+ llvm::Type::getInt64Ty(CGM.getLLVMContext()), Ty->getNumRows()));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ ColumnCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ Subscripts.push_back(DBuilder.getOrCreateSubrange(
+ RowCountNode /*count*/, nullptr /*lowerBound*/, nullptr /*upperBound*/,
+ nullptr /*stride*/));
+ llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts);
+ return DBuilder.createArrayType(Size, Align, ElementTy, SubscriptArray);
+}
+
llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
uint64_t Size;
uint32_t Align;
@@ -4295,6 +4322,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
return CreateType(cast<VectorType>(Ty), Unit);
case Type::ConstantMatrix:
return CreateType(cast<ConstantMatrixType>(Ty), Unit);
+ case Type::CooperativeMatrix:
+ return CreateType(cast<CooperativeMatrixType>(Ty), Unit);
case Type::ObjCObjectPointer:
return CreateType(cast<ObjCObjectPointerType>(Ty), Unit);
case Type::ObjCObject:
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index a818d6471ff3e..ceaa8b7959544 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -239,6 +239,7 @@ class CGDebugInfo {
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ConstantMatrixType *Ty, llvm::DIFile *F);
+ llvm::DIType *CreateType(const CooperativeMatrixType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 9201e40bc13a1..a0091fd5d90a3 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7522,3 +7522,64 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
}
+
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
+ llvm::Value *ScalarVal,
+ QualType CoopMatQTy)
+{
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(
+ VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(
+ VecTy->getElementCount(),
+ ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
+ BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy)
+{
+ auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
+
+ switch (Opcode) {
+ case BO_Add:
+ return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
+ : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+
+ case BO_Sub:
+ return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
+ : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+
+ case BO_Mul:
+ // mat * mat → element-wise FMul/IMul
+ // mat * scalar → handled separately (see scalar path below)
+ if (!RHS->getType()->isVectorTy()) {
+ // mat * scalar: broadcast scalar then element-wise mul
+ llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
+ return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
+ : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ }
+ return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
+ : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+
+ case BO_Div:
+ if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
+
+ default:
+ llvm_unreachable("Unsupported cooperative matrix binary op");
+ }
+}
\ No newline at end of file
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 8783b43846434..3eb21445b883f 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -951,6 +951,12 @@ class ScalarExprEmitter
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
+ if (E->getType()->isCooperativeMatrixType()) { \
+ return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
+ CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), \
+ E->getType()); \
+ } \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
if (result && !promotionTy.isNull()) \
@@ -3688,6 +3694,14 @@ Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
+ if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
+ QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ if (CompTy->isFloatingType())
+ return Builder.CreateFNeg(Val, "coopmat.fneg");
+ else
+ return Builder.CreateNeg(Val, "coopmat.ineg");
+ }
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
: PromotionType;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 119aebb673789..6562bcaf1eeda 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -277,6 +277,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::FunctionProto:
case Type::FunctionNoProto:
case Type::Enum:
@@ -2569,6 +2570,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Record:
case Type::Enum:
case Type::Using:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 3fc9052adb87b..a6e1679713437 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -5096,6 +5096,15 @@ class CodeGenFunction : public CodeGenTypeCache {
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
+ /// Helper function for EmitCoopMatBinaryOp
+ llvm::Value *EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy);
+
+ /// EmitCoopMatBinaryOp - Emit the computation of the specified binary op,
+ /// returning the result.
+ llvm::Value *EmitCoopMatBinaryOp(BinaryOperatorKind Opcode, llvm::Value *LHS,
+ llvm::Value *RHS, QualType ResultTy);
+
/// Emit a conversion from the specified type to the specified destination
/// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 1854df7c7c0f1..ecd32fb65e9a1 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -331,7 +331,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
// Accesses to matrix types are accesses to objects of their element types.
if (const auto *MTy = dyn_cast<MatrixType>(Ty)) {
- assert(isa<ConstantMatrixType>(Ty) &&
+ assert((isa<ConstantMatrixType>(Ty) || isa<CooperativeMatrixType>(Ty)) &&
"only ConstantMatrixType should reach CodeGen");
return getTypeInfo(MTy->getElementType());
}
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index 55fe216580314..b47460540bb53 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -120,7 +120,21 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
}
return llvm::ArrayType::get(IRElemTy, MT->getNumElementsFlattened());
}
-
+ if (T->isCooperativeMatrixType()) {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent a Coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+ return CoopMatType;
+ }
llvm::Type *R = ConvertType(T);
// Check for the boolean vector case.
@@ -684,6 +698,23 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
MT->getNumRows() * MT->getNumColumns());
break;
}
+ case Type::CooperativeMatrix: {
+ const CooperativeMatrixType *DMT =
+ cast<CooperativeMatrixType>(T->getUnqualifiedDesugaredType());
+ llvm::Type *ElTy = ConvertType(DMT->getElementType());
+ // Type argument for TargetExtType
+ llvm::Type *ArgTys[] = {ElTy};
+ // Unsigned arguments for TargetExtType
+ unsigned Ints[] = {DMT->getScope(), DMT->getNumRows(), DMT->getNumColumns(),
+ DMT->getUse()};
+ // Create a TargetExtType to represent the coop matrix type
+ llvm::TargetExtType *CoopMatType = llvm::TargetExtType::get(
+ getLLVMContext(), "spirv.CooperativeMatrixKHR",
+ llvm::ArrayRef<llvm::Type *>(ArgTys), llvm::ArrayRef<unsigned>(Ints));
+
+ ResultType = CoopMatType;
+ break;
+ }
case Type::FunctionNoProto:
case Type::FunctionProto:
ResultType = ConvertFunctionTypeInternal(T);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 5c5fefe32c06c..ad318f73ab19e 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -4017,6 +4017,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::Atomic:
// FIXME: GCC treats block pointers as fundamental types?!
@@ -4290,6 +4291,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
case Type::Vector:
case Type::ExtVector:
case Type::ConstantMatrix:
+ case Type::CooperativeMatrix:
case Type::Complex:
case Type::BlockPointer:
// Itanium C++ ABI 2.9.5p4:
diff --git a/clang/lib/CodeGen/QualTypeMapper.cpp b/clang/lib/CodeGen/QualTypeMapper.cpp
index 31d9250a48ec9..a0f1e2214465f 100644
--- a/clang/lib/CodeGen/QualTypeMapper.cpp
+++ b/clang/lib/CodeGen/QualTypeMapper.cpp
@@ -108,6 +108,12 @@ const llvm::abi::Type *QualTypeMapper::convertTypeImpl(QualType QT) {
MT->getNumRows() * MT->getNumColumns(),
ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
}
+ case Type::CooperativeMatrix: {
+ const auto *MT = cast<CooperativeMatrixType>(QT);
+ return Builder.getArrayType(convertType(MT->getElementType()),
+ MT->getNumRows() * MT->getNumColumns(),
+ ASTCtx.getTypeSize(QT), /*IsMatrixType=*/true);
+ }
case Type::MemberPointer:
return convertMemberPointerType(cast<MemberPointerType>(QT));
case Type::BitInt: {
diff --git a/clang/lib/CodeGen/QualTypeMapper.h b/clang/lib/CodeGen/QualTypeMapper.h
index 35876f44f3aba..76073e980d576 100644
--- a/clang/lib/CodeGen/QualTypeMapper.h
+++ b/clang/lib/CodeGen/QualTypeMapper.h
@@ -47,6 +47,7 @@ class QualTypeMapper {
const llvm::abi::Type *
convertMemberPointerType(const clang::MemberPointerType *MPT);
const llvm::abi::Type *convertMatrixType(const ConstantMatrixType *MT);
+ const llvm::abi::Type *convertMatrixType(const CooperativeMatrixType *CMT);
const llvm::abi::RecordType *convertStructType(const clang::RecordDecl *RD);
const llvm::abi::RecordType *convertUnionType(const clang::RecordDecl *RD);
>From b3a04ef78ce18d1d828630bd312b28184cf9fb3c Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:20:46 -0400
Subject: [PATCH 06/12] Add LIT tests and fix minor build issues
---
clang/include/clang/AST/TypeBase.h | 8 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 94 +++++++++++++++
.../test/Preprocessor/coop-mat-opencl-ext.cl | 63 ++++++++++
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 66 ++++++++++
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 60 +++++++++
clang/test/SemaOpenCL/coop-mat-sema.cl | 83 +++++++++++++
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 114 ++++++++++++++++++
clang/tools/libclang/CIndex.cpp | 1 +
llvm/test/Verifier/coop-mat-verifier-fix.ll | 39 ++++++
9 files changed, 524 insertions(+), 4 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen.cl
create mode 100644 clang/test/Preprocessor/coop-mat-opencl-ext.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-ast-utils.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema-neg.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-sema.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-type-infra.cl
create mode 100644 llvm/test/Verifier/coop-mat-verifier-fix.ll
diff --git a/clang/include/clang/AST/TypeBase.h b/clang/include/clang/AST/TypeBase.h
index 5ed2039adb816..f65affa5f3bdd 100644
--- a/clang/include/clang/AST/TypeBase.h
+++ b/clang/include/clang/AST/TypeBase.h
@@ -4617,14 +4617,14 @@ class CooperativeMatrixType final : public MatrixType {
/// Return true if \p Scope is valid
static constexpr bool isScopeValid(size_t Scope) {
- return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM */;
+ return Scope == 3 /* CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP */;
}
/// Return true if \p Use is valid
static constexpr bool isUseValid(size_t Use) {
- return Use == 0 /* CLK_COOPERATIVE_MATRIX_A_QCOM */ ||
- Use == 1 /* CLK_COOPERATIVE_MATRIX_B_QCOM */ ||
- Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM */;
+ return Use == 0 /* CLK_COOPERATIVE_MATRIX_A */ ||
+ Use == 1 /* CLK_COOPERATIVE_MATRIX_B */ ||
+ Use == 2 /* CLK_COOPERATIVE_MATRIX_ACCUMULATOR */;
}
/// Returns the maximum number of elements per dimension.
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
new file mode 100644
index 0000000000000..58b7a882fdc87
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -0,0 +1,94 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+//
+// Patch 5: CodeGen -- TargetExtType lowering, builtin IR emission,
+// SPIR-V intrinsic names.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 5a. coop_mat_load -> __spirv_CooperativeMatrixLoadKHR
+// Also verifies CooperativeMatrixType lowers to spirv.CooperativeMatrixKHR
+// TargetExtType (visible in the call signature).
+// ---------------------------------------------------------------------------
+kernel void test_load(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load
+// CHECK: call
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CooperativeMatrixLoadKHR
+
+// ---------------------------------------------------------------------------
+// 5b. coop_mat_store -> __spirv_CooperativeMatrixStoreKHR
+// ---------------------------------------------------------------------------
+kernel void test_store(__global float *ptr, MatA_t a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR
+
+// ---------------------------------------------------------------------------
+// 5c. coop_mat_mulAdd -> __spirv_CooperativeMatrixMulAddKHR
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a; MatB_t b; MatC_t c; MatC_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+
+// ---------------------------------------------------------------------------
+// 5d. Binary add (float element) -> __spirv_CooperativeMatrixFAdd
+// ---------------------------------------------------------------------------
+kernel void test_binary_add(__global float *ptr) {
+ MatA_t a; MatA_t b; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a + b;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_binary_add
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixFAdd
+
+// ---------------------------------------------------------------------------
+// 5e. Scalar multiply -> __spirv_CooperativeMatrixScalarMulKHR
+// ---------------------------------------------------------------------------
+kernel void test_scalar_mul(__global float *ptr, float s) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = a * s;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_scalar_mul
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarMulKHR
+
+// ---------------------------------------------------------------------------
+// 5f. Unary minus -> __spirv_CooperativeMatrixScalarNeg
+// ---------------------------------------------------------------------------
+kernel void test_unary_neg(__global float *ptr) {
+ MatA_t a; MatA_t r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = -a;
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_unary_neg
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixScalarNeg
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
new file mode 100644
index 0000000000000..7181efbe7e2a8
--- /dev/null
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -0,0 +1,63 @@
+// clang/test/Preprocessor/coop_mat_opencl_ext.cl
+//
+// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
+//
+// Tests: extension macro is predefined when enabled, extension can be
+// explicitly enabled/disabled via pragma, all four enum types and
+// their constants are visible under -finclude-default-header.
+
+// ── 2a. Extension macro is predefined when the extension is enabled ─────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E -dM %s \
+// RUN: | FileCheck %s --check-prefix=EXT
+
+// EXT: cl_ext_kernel_cooperative_matrix
+
+// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -E %s \
+// RUN: | FileCheck %s --check-prefix=NOEXT
+
+// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+
+// ── 2c. Enum constants are visible when extension is enabled ─────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+void test_enum_constants(void) {
+ // coop_matrix_scope_t
+ coop_matrix_scope_t s = CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP;
+ (void)s;
+
+ // coop_matrix_use_t
+ coop_matrix_use_t u0 = CLK_COOPERATIVE_MATRIX_A;
+ coop_matrix_use_t u1 = CLK_COOPERATIVE_MATRIX_B;
+ coop_matrix_use_t u2 = CLK_COOPERATIVE_MATRIX_ACCUMULATOR;
+ (void)u0; (void)u1; (void)u2;
+
+ // coop_matrix_layout_t
+ coop_matrix_layout_t l0 = CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR;
+ coop_matrix_layout_t l1 = CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR;
+ (void)l0; (void)l1;
+
+ // coop_matrix_operands_t
+ coop_matrix_operands_t op = CLK_COOPERATIVE_MATRIX_OPERAND_NONE;
+ (void)op;
+}
+
+// ── 2d. Enum constant values match the spec ──────────────────────────────────
+void test_enum_values(void) {
+ _Static_assert(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP == 3, "scope subgroup");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_A == 0, "use A");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_B == 1, "use B");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_ACCUMULATOR == 2, "use ACC");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR == 0, "row major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR== 1, "col major");
+ _Static_assert(CLK_COOPERATIVE_MATRIX_OPERAND_NONE == 0, "operand none");
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
new file mode 100644
index 0000000000000..c6230eb94531f
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -0,0 +1,66 @@
+// clang/test/SemaOpenCL/coop_mat_ast_utils.cl
+//
+// Patch 3: AST utility support — ASTImporter, ASTStructuralEquivalence,
+// ExprConstant, ItaniumMangle, MicrosoftMangle,
+// ASTReader/ASTWriter TypeLoc serialisation.
+//
+// ── 3a. Itanium mangling ────────────────────────────────────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=MANGLE
+//
+// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch \
+// RUN: -ast-dump %t.pch_probe.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+//
+// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ── 3a. Mangling — function with a coop-mat parameter gets a mangled name
+// that contains the vendor-extended "coop_mat" marker.
+kernel void test_mangling(MatA_t a) { (void)a; }
+
+// MANGLE: @{{.*}}test_mangling{{.*}}(
+
+// ── 3b. PCH — after the round-trip the FunctionDecl for pch_probe
+// must still carry a coop_mat parameter type.
+// PCH: FunctionDecl {{.*}} pch_probe
+// PCH: ParmVarDecl {{.*}} a {{.*}}coop_mat(
+
+// ── 3c. Structural equivalence — same parameters produce no diagnostic.
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_structural_equiv(void) {
+ MatA_t *p = 0;
+ MatA_alias *q = p; // same canonical type
+ (void)q;
+}
+
+// ── 3d. ExprConstant — coop_mat type classified as "no class" (not an
+// integer, float, pointer …). Using __builtin_classify_type on it
+// compiles without error; result is 0 (no_type_class).
+void test_expr_constant(MatA_t a) {
+ int cls = __builtin_classify_type(a);
+ (void)cls;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
new file mode 100644
index 0000000000000..f8080448b6df5
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -0,0 +1,60 @@
+// clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+//
+// Patch 4: Sema — negative tests for diagnostic paths.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. Invalid scope value (0 is not CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(0, 16, 16, USE_A))) MatBadScope; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 2. Invalid use value (99 is not 0/1/2)
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, 99))) MatBadUse; // expected-error {{invalid argument of cooperative matrix attribute}}
+
+// ---------------------------------------------------------------------------
+// 3. Mismatched element types in coop_mat_mulAdd
+// a/b are float matrices, c is an int matrix — should fire element type
+// mismatch diagnostic.
+// Use the two-step declare-then-assign pattern so that the coop_mat_load
+// fixup path fires correctly and we only get the intended mulAdd error.
+// ---------------------------------------------------------------------------
+kernel void test_muladd_type_mismatch(__global float *fptr,
+ __global int *iptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_int_t c;
+ a = coop_mat_load(fptr, ROW_MAJOR, 16);
+ b = coop_mat_load(fptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+
+ MatC_int_t result;
+ result = coop_mat_mulAdd(a, b, c); // expected-error {{inconsistent cooperative matrix element type}}
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4. Assignment of coop_mat_load result to a plain scalar — must fire the
+// "should be assigned to cooperative matrix type variable" diagnostic.
+// This intentionally uses single-step initialization because the error
+// fires precisely when the LHS is NOT a coop mat type.
+// ---------------------------------------------------------------------------
+kernel void test_bad_assignment(__global float *ptr) {
+ float bad;
+ bad = coop_mat_load(ptr, ROW_MAJOR, 16); // expected-error {{builtin return value should be assigned to cooperative matrix type variable}}
+ (void)bad;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
new file mode 100644
index 0000000000000..cec1f278b0037
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -0,0 +1,83 @@
+// clang/test/SemaOpenCL/coop-mat-sema.cl
+//
+// Patch 4: Sema -- BuildCooperativeMatrixType, diagnostics,
+// builtin validation, expr handling, TreeTransform.
+//
+// Valid code -- expected-no-diagnostics
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_t;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_t;
+
+// ---------------------------------------------------------------------------
+// 4b. load builtin -- return type is fixed up by AddInitializerToDecl to
+// match the LHS variable type. Declare first, then assign so that
+// Sema's IsCoopMatrixBuiltin path fires correctly.
+// ---------------------------------------------------------------------------
+kernel void test_load_store(__global float *ptr,
+ __global float *out_ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ coop_mat_store(out_ptr, a, ROW_MAJOR, 16);
+}
+
+// ---------------------------------------------------------------------------
+// 4c. mulAdd builtin -- same two-step pattern for each matrix.
+// ---------------------------------------------------------------------------
+kernel void test_muladd(__global float *ptr) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ MatC_t result;
+ result = coop_mat_mulAdd(a, b, c);
+ (void)result;
+}
+
+// ---------------------------------------------------------------------------
+// 4d. Binary element-wise operators (+, -)
+// ---------------------------------------------------------------------------
+void test_binary_ops(MatA_t a, MatA_t b) {
+ MatA_t r_add = a + b;
+ MatA_t r_sub = a - b;
+ (void)r_add; (void)r_sub;
+}
+
+// ---------------------------------------------------------------------------
+// 4e. Scalar multiply operator
+// ---------------------------------------------------------------------------
+void test_scalar_ops(MatA_t a, float s) {
+ MatA_t r = a * s;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4f. Unary minus
+// ---------------------------------------------------------------------------
+void test_unary_minus(MatA_t a) {
+ MatA_t r = -a;
+ (void)r;
+}
+
+// ---------------------------------------------------------------------------
+// 4g. Assignment -- coop_mat_load return assigned to a pre-declared var.
+// This exercises the SemaDecl AddInitializerToDecl fixup path directly.
+// ---------------------------------------------------------------------------
+kernel void test_assignment(__global float *ptr) {
+ MatA_t a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
new file mode 100644
index 0000000000000..2e6700973e611
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -0,0 +1,114 @@
+// clang/test/SemaOpenCL/coop_mat_type_infra.cl
+//
+// Patch 1: CooperativeMatrixType AST node and type infrastructure.
+//
+// Tests: TypeNodes.td registration, CooperativeMatrixType class accessors,
+// ASTContext::getCooperativeMatrixType uniquing, TypePrinter,
+// RecursiveASTVisitor traversal, mergeTypes compatibility,
+// TypeLoc operand slots, sizeof / getTypeInfoImpl.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -ast-dump %s \
+// RUN: | FileCheck %s --check-prefix=AST
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
+// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
+// RUN: | FileCheck %s --check-prefix=PCH
+
+// ---------------------------------------------------------------------------
+// Enum constants (from opencl-c-base.h via -finclude-default-header)
+// ---------------------------------------------------------------------------
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+
+// ---------------------------------------------------------------------------
+// 1. Basic type construction — four use roles, float element type.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatB_float16x16;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatC_float16x16;
+typedef int __attribute__((coop_mat(SCOPE, 8, 8, USE_A))) MatA_int8x8;
+
+// AST: TypedefDecl {{.*}} MatA_float16x16
+// AST: TypedefDecl {{.*}} MatB_float16x16
+// AST: TypedefDecl {{.*}} MatC_float16x16
+// AST: TypedefDecl {{.*}} MatA_int8x8
+
+// ---------------------------------------------------------------------------
+// 2. Type printer — VarDecls carry the coop_mat attribute in their type string.
+// ---------------------------------------------------------------------------
+void test_type_spelling(void) {
+ MatA_float16x16 a;
+ MatB_float16x16 b;
+ MatC_float16x16 c;
+ MatA_int8x8 d;
+}
+
+// AST: VarDecl {{.*}} a {{.*}}coop_mat(
+// AST: VarDecl {{.*}} b {{.*}}coop_mat(
+// AST: VarDecl {{.*}} c {{.*}}coop_mat(
+// AST: VarDecl {{.*}} d {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 3. sizeof / getTypeInfoImpl — width = elem * rows * cols
+// float(4B)*16*16 = 1024 B = 8192 bits
+// int(4B)*8*8 = 256 B = 2048 bits
+// ---------------------------------------------------------------------------
+void test_sizeof(void) {
+ _Static_assert(sizeof(MatA_float16x16) == 1024, "float 16x16 size");
+ _Static_assert(sizeof(MatA_int8x8) == 256, "int 8x8 size");
+}
+
+// ---------------------------------------------------------------------------
+// 4. Parameter / return type — type preserved through function boundary.
+// ---------------------------------------------------------------------------
+MatA_float16x16 test_param_return(MatA_float16x16 in) {
+ return in;
+}
+
+// AST: FunctionDecl {{.*}} test_param_return
+// AST: ParmVarDecl {{.*}} in {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 5. TypeLoc operand traversal — all four operand slots populated.
+// ---------------------------------------------------------------------------
+void test_typeloc_operands(void) {
+ float __attribute__((coop_mat(SCOPE, 4, 8, USE_C))) local_acc;
+ (void)local_acc;
+}
+
+// AST: VarDecl {{.*}} local_acc {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 6. RecursiveASTVisitor — element type (half) reachable through the node.
+// ---------------------------------------------------------------------------
+void test_visitor_element_type(half __attribute__((coop_mat(SCOPE, 8, 8, USE_B))) x) {
+ (void)x;
+}
+
+// AST: ParmVarDecl {{.*}} x {{.*}}coop_mat(
+
+// ---------------------------------------------------------------------------
+// 7. mergeTypes / type compatibility — two identical typedefs resolve to the
+// same canonical type; taking a pointer across them compiles cleanly.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatA_alias;
+
+void test_merge_types(void) {
+ MatA_float16x16 *p = 0;
+ MatA_alias *q = p; // same canonical type — no diagnostic
+ (void)q;
+}
+
+// ---------------------------------------------------------------------------
+// 8. PCH serialisation round-trip.
+// ---------------------------------------------------------------------------
+// PCH: VarDecl {{.*}} g {{.*}}coop_mat(
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 7eeeba9c7948d..71e3ed66d666a 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1897,6 +1897,7 @@ DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
DEFAULT_TYPELOC_IMPL(Vector, Type)
DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
DEFAULT_TYPELOC_IMPL(ConstantMatrix, MatrixType)
+DEFAULT_TYPELOC_IMPL(CooperativeMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(DependentSizedMatrix, MatrixType)
DEFAULT_TYPELOC_IMPL(FunctionProto, FunctionType)
DEFAULT_TYPELOC_IMPL(FunctionNoProto, FunctionType)
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
new file mode 100644
index 0000000000000..c4e206b76b760
--- /dev/null
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -0,0 +1,39 @@
+; llvm/test/Verifier/coop_mat_verifier_fix.ll
+;
+; Patch 6: Fix operator-precedence bug in Verifier.cpp intrinsic check.
+;
+; The bug: (A && B <= 64) was parsed as (A && (B <= 64)) instead of
+; ((A && B) <= 64). After the fix the parentheses are explicit.
+;
+; This test verifies that a valid @llvm.experimental.noalias.scope.decl
+; call with a 2-element scope list (no third argument) passes the verifier
+; without error, and that an invalid third argument (> 64 bits) is caught.
+;
+; ── 6a. Valid call — no third argument → verifier must accept ────────────────
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s --check-prefix=VALID
+; RUN: llvm-as < %s | opt -passes=verify -disable-output
+;
+; ── 6b. Pipe through the verifier explicitly ─────────────────────────────────
+; RUN: llvm-as %s -o %t.bc
+; RUN: opt -passes=verify %t.bc -disable-output
+
+; VALID: @llvm.experimental.noalias.scope.decl
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+define void @test_no_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+; Third arg present and <= 64 bits wide — must also pass
+define void @test_valid_third_arg() {
+ %domain = call token @llvm.experimental.noalias.scope.decl(
+ metadata !0)
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain_a"}
>From 16a342b4e1f5fde6ef3daf2f523d34c4465829d7 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Fri, 10 Jul 2026 18:39:15 -0400
Subject: [PATCH 07/12] Fix formatting issues
---
clang/include/clang/Sema/Sema.h | 15 +++---
clang/lib/CodeGen/CGBuiltin.cpp | 76 +++++++++++++++------------
clang/lib/CodeGen/CGExpr.cpp | 63 ++++++++++------------
clang/lib/CodeGen/CGExprScalar.cpp | 12 ++---
clang/lib/Sema/SemaChecking.cpp | 39 +++++++-------
clang/lib/Sema/SemaExpr.cpp | 35 ++++++------
clang/lib/Sema/TreeTransform.h | 2 +-
clang/lib/Serialization/ASTReader.cpp | 3 +-
clang/lib/Serialization/ASTWriter.cpp | 3 +-
9 files changed, 120 insertions(+), 128 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9ef1bf75878d3..84c2d0d01240f 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3068,19 +3068,16 @@ class Sema final : public SemaBase {
// Coop matrix handling.
void CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc, CallExpr *call);
+ SourceLocation MatrixLoc, CallExpr *call);
void CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc);
+ QualType BufferType,
+ SourceLocation MatrixLoc);
bool CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx);
bool CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr);
- ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall,
- ExprResult CallResult);
- ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixStore(CallExpr *TheCall, ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixLoad(CallExpr *TheCall, ExprResult CallResult);
void CheckCoopMatrixMatMulOutput(CallExpr *TheCall);
- ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult);
+ ExprResult BuiltinCoopMatrixMulAdd(CallExpr *TheCall, ExprResult CallResult);
ExprResult CreateCoopMatBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc,
Expr *LHSExpr, Expr *RHSExpr);
bool CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 676ee0cc5b4ad..fa48fc0eb2458 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2980,7 +2980,7 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
llvm::Type *Tys[] = {ElTy};
// Unsigned arguments for TargetExtType
unsigned Ints[] = {MTy->getScope(), MTy->getNumRows(), MTy->getNumColumns(),
- MTy->getUse()};
+ MTy->getUse()};
// Create a TargetExtType to represent the coop matrix type
llvm::TargetExtType *RetType = llvm::TargetExtType::get(
CGM.getLLVMContext(), "spirv.CooperativeMatrixKHR",
@@ -4854,8 +4854,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_load without coop_mat output operand");
+ CGM.ErrorUnsupported(E, "coop_mat_load without coop_mat output operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Layout = EmitScalarExpr(E->getArg(1));
@@ -4869,7 +4868,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LoadFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LoadFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LoadFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LoadFn, {Ptr, Layout, Stride});
@@ -4886,8 +4886,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
E->getArg(0)->getExprLoc(), FD, 0);
const auto *MTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_store without coop_mat input operand");
+ CGM.ErrorUnsupported(E, "coop_mat_store without coop_mat input operand");
auto Ptr = EmitScalarExpr(E->getArg(0));
auto Arg0 = EmitScalarExpr(E->getArg(1));
@@ -4897,11 +4896,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::TargetExtType *ArgType = getTargetExtType(*this, CGM, MTy);
// Set function type.
llvm::FunctionType *FTy = llvm::FunctionType::get(
- VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()}, false);
+ VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
+ false);
// Function name mangling.
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee StoreFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee StoreFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(StoreFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(StoreFn, {Ptr, Arg0, Layout, Stride});
@@ -4912,8 +4913,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_mulAdd: {
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
if (!MTy)
- CGM.ErrorUnsupported(
- E, "coop_mat_mulAdd without coop_mat output operand");
+ CGM.ErrorUnsupported(E,
+ "coop_mat_mulAdd without coop_mat output operand");
auto Arg0 = E->getArg(0);
auto Arg1 = E->getArg(1);
@@ -4937,11 +4938,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto *AType = getTargetExtType(*this, CGM, MATy);
auto *BType = getTargetExtType(*this, CGM, MBTy);
auto *CType = getTargetExtType(*this, CGM, MCTy);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, CType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee MatMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee MatMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(MatMulFn, {MA, MB, MC, isDataSigned});
@@ -4955,7 +4957,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BIcoop_mat_binary_div: {
auto Arg0 = EmitScalarExpr(E->getArg(0));
auto Arg1 = EmitScalarExpr(E->getArg(1));
-
+
const auto *MTy = E->getType()->getAs<CooperativeMatrixType>();
const auto *MATy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
const auto *MBTy = E->getArg(1)->getType()->getAs<CooperativeMatrixType>();
@@ -4972,13 +4974,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
-
- std::string Name = (ElTy->isIntegerTy())
- ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
- : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
- llvm::FunctionCallee BinaryFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
+
+ std::string Name =
+ (ElTy->isIntegerTy())
+ ? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
+ : getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ llvm::FunctionCallee BinaryFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(BinaryFn, {Arg0, Arg1, isDataSigned});
@@ -5004,10 +5008,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, BType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarMulFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarMulFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarMulFn, {Arg0, Arg1, isDataSigned});
@@ -5031,10 +5036,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
llvm::Value *isDataSigned = llvm::ConstantInt::get(Int1Ty, isSigned);
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {AType, Int1Ty}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee ScalarNegFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee ScalarNegFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(ScalarNegFn, {Arg0, isDataSigned});
@@ -5047,10 +5053,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Init = EmitScalarExpr(E->getArg(0));
auto *RetType = getTargetExtType(*this, CGM, MTy);
// Set function type.
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- RetType, {Init->getType()}, false);
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(RetType, {Init->getType()}, false);
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee InitFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee InitFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(InitFn, {Init});
@@ -5062,11 +5069,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
const auto *MTy = E->getArg(0)->getType()->getAs<CooperativeMatrixType>();
auto *AType = getTargetExtType(*this, CGM, MTy);
auto Arg0 = EmitScalarExpr(E->getArg(0));
- llvm::FunctionType *FTy = llvm::FunctionType::get(
- ConvertType(E->getType()), {AType}, false);
-
+ llvm::FunctionType *FTy =
+ llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
+
std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
- llvm::FunctionCallee LengthFn = CGM.getModule().getOrInsertFunction(Name, FTy);
+ llvm::FunctionCallee LengthFn =
+ CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
auto *NewCall = Builder.CreateCall(LengthFn, {Arg0});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index a0091fd5d90a3..432b5927c4015 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -7523,44 +7523,37 @@ void CodeGenFunction::FlattenAccessAndTypeLValue(
}
}
-llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(
- llvm::Value *ScalarVal,
- QualType CoopMatQTy)
-{
- llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
- auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
-
- // Fast path for compile-time constants
- if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
- return llvm::ConstantVector::getSplat(
- VecTy->getElementCount(), C);
-
- // Runtime path: splat scalar across all vector lanes
- return Builder.CreateVectorSplat(
- VecTy->getElementCount(),
- ScalarVal,
- "coopmat.broadcast");
-}
-
-llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
- BinaryOperatorKind Opcode,
- llvm::Value *LHS,
- llvm::Value *RHS,
- QualType ResultTy)
-{
+llvm::Value *CodeGenFunction::EmitCoopMatFromScalar(llvm::Value *ScalarVal,
+ QualType CoopMatQTy) {
+ llvm::Type *CoopMatLLVMTy = ConvertType(CoopMatQTy);
+ auto *VecTy = cast<llvm::FixedVectorType>(CoopMatLLVMTy);
+
+ // Fast path for compile-time constants
+ if (auto *C = dyn_cast<llvm::Constant>(ScalarVal))
+ return llvm::ConstantVector::getSplat(VecTy->getElementCount(), C);
+
+ // Runtime path: splat scalar across all vector lanes
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), ScalarVal,
+ "coopmat.broadcast");
+}
+
+llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(BinaryOperatorKind Opcode,
+ llvm::Value *LHS,
+ llvm::Value *RHS,
+ QualType ResultTy) {
auto *CoopMatTy = ResultTy->getAs<CooperativeMatrixType>();
- QualType CompTy = CoopMatTy->getElementType();
- bool IsFloat = CompTy->isFloatingType();
- bool IsSigned = CompTy->isSignedIntegerType();
+ QualType CompTy = CoopMatTy->getElementType();
+ bool IsFloat = CompTy->isFloatingType();
+ bool IsSigned = CompTy->isSignedIntegerType();
switch (Opcode) {
case BO_Add:
return IsFloat ? Builder.CreateFAdd(LHS, RHS, "coopmat.fadd")
- : Builder.CreateAdd (LHS, RHS, "coopmat.iadd");
+ : Builder.CreateAdd(LHS, RHS, "coopmat.iadd");
case BO_Sub:
return IsFloat ? Builder.CreateFSub(LHS, RHS, "coopmat.fsub")
- : Builder.CreateSub (LHS, RHS, "coopmat.isub");
+ : Builder.CreateSub(LHS, RHS, "coopmat.isub");
case BO_Mul:
// mat * mat → element-wise FMul/IMul
@@ -7569,14 +7562,16 @@ llvm::Value *CodeGenFunction::EmitCoopMatBinaryOp(
// mat * scalar: broadcast scalar then element-wise mul
llvm::Value *Broadcast = EmitCoopMatFromScalar(RHS, ResultTy);
return IsFloat ? Builder.CreateFMul(LHS, Broadcast, "coopmat.scalarfmul")
- : Builder.CreateMul (LHS, Broadcast, "coopmat.scalarimul");
+ : Builder.CreateMul(LHS, Broadcast, "coopmat.scalarimul");
}
return IsFloat ? Builder.CreateFMul(LHS, RHS, "coopmat.fmul")
- : Builder.CreateMul (LHS, RHS, "coopmat.imul");
+ : Builder.CreateMul(LHS, RHS, "coopmat.imul");
case BO_Div:
- if (IsFloat) return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
- if (IsSigned) return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
+ if (IsFloat)
+ return Builder.CreateFDiv(LHS, RHS, "coopmat.fdiv");
+ if (IsSigned)
+ return Builder.CreateSDiv(LHS, RHS, "coopmat.sdiv");
return Builder.CreateUDiv(LHS, RHS, "coopmat.udiv");
default:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 3eb21445b883f..a51683128c2bf 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -952,10 +952,9 @@ class ScalarExprEmitter
#define HANDLEBINOP(OP) \
Value *VisitBin##OP(const BinaryOperator *E) { \
if (E->getType()->isCooperativeMatrixType()) { \
- return CGF.EmitCoopMatBinaryOp(E->getOpcode(), \
- CGF.EmitScalarExpr(E->getLHS()), \
- CGF.EmitScalarExpr(E->getRHS()), \
- E->getType()); \
+ return CGF.EmitCoopMatBinaryOp( \
+ E->getOpcode(), CGF.EmitScalarExpr(E->getLHS()), \
+ CGF.EmitScalarExpr(E->getRHS()), E->getType()); \
} \
QualType promotionTy = getPromotionType(E->getType()); \
auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
@@ -3696,11 +3695,12 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
QualType PromotionType) {
if (E->getSubExpr()->getType()->isCooperativeMatrixType()) {
llvm::Value *Val = CGF.EmitScalarExpr(E->getSubExpr());
- QualType CompTy = E->getType()->getAs<CooperativeMatrixType>()->getElementType();
+ QualType CompTy =
+ E->getType()->getAs<CooperativeMatrixType>()->getElementType();
if (CompTy->isFloatingType())
return Builder.CreateFNeg(Val, "coopmat.fneg");
else
- return Builder.CreateNeg(Val, "coopmat.ineg");
+ return Builder.CreateNeg(Val, "coopmat.ineg");
}
QualType promotionTy = PromotionType.isNull()
? getPromotionType(E->getSubExpr()->getType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index ed4ac3721ad7a..7bd653870c5c5 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17310,8 +17310,7 @@ bool Sema::BuiltinNonDeterministicValue(CallExpr *TheCall) {
}
// Check coop_mat_load/store buffer pointer.
-bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
- unsigned PtrArgIdx) {
+bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall, unsigned PtrArgIdx) {
bool ArgError = false;
Expr *PtrExpr = TheCall->getArg(PtrArgIdx);
ExprResult PtrConv = DefaultFunctionArrayLvalueConversion(PtrExpr);
@@ -17342,8 +17341,8 @@ bool Sema::CheckCoopMatrixLoadStorePtr(CallExpr *TheCall,
// Check coop_mat_load/store matrix element has same type with buffer pointer.
void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
- QualType BufferType,
- SourceLocation MatrixLoc) {
+ QualType BufferType,
+ SourceLocation MatrixLoc) {
auto *MTy = MatrixType->getAs<CooperativeMatrixType>();
if (!MTy) {
Diag(MatrixLoc, diag::err_coop_matrix_arg);
@@ -17359,8 +17358,8 @@ void Sema::CheckCoopMatrixLoadStoreElementType(QualType MatrixType,
}
void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
- SourceLocation MatrixLoc,
- CallExpr *call) {
+ SourceLocation MatrixLoc,
+ CallExpr *call) {
FunctionDecl *F = call->getDirectCallee();
assert(F);
@@ -17369,7 +17368,7 @@ void Sema::CheckCoopMatrixLoadElementType(QualType MatrixType,
assert(Fname);
if (Fname->isStr("coop_mat_load"))
CheckCoopMatrixLoadStoreElementType(MatrixType, call->getArg(0)->getType(),
- MatrixLoc);
+ MatrixLoc);
}
// Check coop_mat_load/store layout argument
@@ -17393,7 +17392,7 @@ bool Sema::CheckCoopMatrixLoadStoreLayout(Expr *LayoutExpr) {
}
ExprResult Sema::BuiltinCoopMatrixLoad(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
if (CheckCoopMatrixLoadStorePtr(TheCall, 0))
@@ -17412,7 +17411,7 @@ ExprResult Sema::BuiltinCoopMatrixStore(CallExpr *TheCall,
if (CheckCoopMatrixLoadStorePtr(TheCall, 00))
return ExprError();
CheckCoopMatrixLoadStoreElementType(Arg1->getType(), Arg0->getType(),
- Arg0->getBeginLoc());
+ Arg0->getBeginLoc());
if (CheckCoopMatrixLoadStoreLayout(TheCall->getArg(2)))
return ExprError();
return CallResult;
@@ -17451,7 +17450,7 @@ void Sema::CheckCoopMatrixMatMulOutput(CallExpr *TheCall) {
}
bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
- SourceLocation BLoc) {
+ SourceLocation BLoc) {
auto *M0Ty = ATy->getAs<CooperativeMatrixType>();
auto *M1Ty = BTy->getAs<CooperativeMatrixType>();
if (!M0Ty)
@@ -17480,7 +17479,7 @@ bool Sema::CheckCoopMatrixTypes(QualType ATy, SourceLocation ALoc, QualType BTy,
}
ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17488,7 +17487,7 @@ ExprResult Sema::BuiltinCoopMatrixBinaryOp(CallExpr *TheCall,
Expr *Arg1 = TheCall->getArg(1);
CheckCoopMatrixTypes(Arg0->getType(), Arg0->getBeginLoc(), Arg1->getType(),
- Arg1->getBeginLoc());
+ Arg1->getBeginLoc());
TheCall->setType(Arg0->getType());
@@ -17504,7 +17503,7 @@ static bool isValidMatAMatCElementTypeCombination(QualType ATy, QualType CTy) {
}
ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 3))
return ExprError();
@@ -17552,7 +17551,7 @@ ExprResult Sema::BuiltinCoopMatrixMulAdd(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 2))
return ExprError();
@@ -17563,7 +17562,7 @@ ExprResult Sema::BuiltinCoopMatrixScalarOp(CallExpr *TheCall,
}
ExprResult Sema::BuiltinCoopMatrixScalarUnaryOp(CallExpr *TheCall,
- ExprResult CallResult) {
+ ExprResult CallResult) {
if (checkArgCount(TheCall, 1))
return ExprError();
@@ -17596,17 +17595,17 @@ ExprResult Sema::BuiltinMatrixTranspose(CallExpr *TheCall,
// matrix type.
if (ConstMType) {
QualType ResultType = Context.getConstantMatrixType(
- ConstMType->getElementType(), ConstMType->getNumColumns(),
- ConstMType->getNumRows());
+ ConstMType->getElementType(), ConstMType->getNumColumns(),
+ ConstMType->getNumRows());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
}
if (CoopMType) {
QualType ResultType = Context.getCooperativeMatrixType(
- CoopMType->getElementType(),CoopMType->getScope(),
- CoopMType->getNumColumns(), CoopMType->getNumRows(),
- CoopMType->getUse());
+ CoopMType->getElementType(), CoopMType->getScope(),
+ CoopMType->getNumColumns(), CoopMType->getNumRows(),
+ CoopMType->getUse());
// Change the return type to the type of the returned matrix.
TheCall->setType(ResultType);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 333212ef369a1..7d86d6e34e24e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -11294,8 +11294,7 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
if (LHSTy->isSveVLSBuiltinType() || RHSTy->isSveVLSBuiltinType())
return CheckSizelessVectorOperands(LHS, RHS, Loc, IsCompAssign,
ArithConvKind::Arithmetic);
- if (!IsDiv &&
- (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
+ if (!IsDiv && (LHSTy->isMatrixType() || RHSTy->isMatrixType()))
return CheckMatrixMultiplyOperands(LHS, RHS, Loc, IsCompAssign);
// For division, only matrix-by-scalar is supported. Other combinations with
// matrix types are invalid.
@@ -13844,10 +13843,11 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
return QualType();
if (LHS.get()->getType()->isCooperativeMatrixType() ||
- RHS.get()->getType()->isCooperativeMatrixType()) {
+ RHS.get()->getType()->isCooperativeMatrixType()) {
auto *LHSMatType = LHS.get()->getType()->getAs<CooperativeMatrixType>();
auto *RHSMatType = RHS.get()->getType()->getAs<CooperativeMatrixType>();
- assert((LHSMatType || RHSMatType) && "At least one operand must be a matrix");
+ assert((LHSMatType || RHSMatType) &&
+ "At least one operand must be a matrix");
if (LHSMatType && RHSMatType) {
if (LHSMatType->getNumColumns() != RHSMatType->getNumRows())
return InvalidOperands(Loc, LHS, RHS);
@@ -13858,14 +13858,14 @@ QualType Sema::CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS,
RHS.get()->getType().getUnqualifiedType());
QualType LHSELTy = LHSMatType->getElementType(),
- RHSELTy = RHSMatType->getElementType();
+ RHSELTy = RHSMatType->getElementType();
if (!Context.hasSameType(LHSELTy, RHSELTy))
return InvalidOperands(Loc, LHS, RHS);
return Context.getCooperativeMatrixType(
- Context.getCommonSugaredType(LHSELTy, RHSELTy), LHSMatType->getScope(),
- LHSMatType->getNumRows(), RHSMatType->getNumColumns(),
- LHSMatType->getUse());
+ Context.getCommonSugaredType(LHSELTy, RHSELTy),
+ LHSMatType->getScope(), LHSMatType->getNumRows(),
+ RHSMatType->getNumColumns(), LHSMatType->getUse());
}
}
@@ -15589,17 +15589,13 @@ ExprResult Sema::CreateCoopMatBinOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Add:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_add, Args);
case BO_Sub:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_sub, Args);
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_mul, Args);
case BO_Div:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_binary_div, Args);
default:
break;
}
@@ -15614,8 +15610,7 @@ ExprResult Sema::CreateCoopMatScalarOp(SourceLocation OpLoc,
Args.push_back(RHSExpr);
switch (Opc) {
case BO_Mul:
- return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul,
- Args);
+ return BuildBuiltinCallExpr(OpLoc, Builtin::BIcoop_mat_scalar_mul, Args);
default:
break;
}
@@ -15663,7 +15658,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
// Check matrix types for assignment.
if (BO_Assign == Opc) {
if (CheckCoopMatrixTypes(LHSTy, LHSExpr->getBeginLoc(), RHSTy,
- RHSExpr->getBeginLoc()))
+ RHSExpr->getBeginLoc()))
return ExprError();
} else
return CreateCoopMatBinOp(OpLoc, Opc, LHSExpr, RHSExpr);
@@ -15705,7 +15700,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
assert(call);
call->setType(LHSExpr->getType());
CheckCoopMatrixLoadElementType(LHSExpr->getType(), LHSExpr->getBeginLoc(),
- call);
+ call);
CheckCoopMatrixMatMulOutput(call);
}
ResultTy = CheckAssignmentOperands(LHS.get(), RHS, OpLoc, QualType(), Opc);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2e7b652c244f6..09541de674d03 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1049,7 +1049,7 @@ class TreeTransform {
QualType RebuildCooperativeMatrixType(QualType ElementType, unsigned Scope,
unsigned NumRows, unsigned NumColumns,
unsigned Use);
-
+
/// Build a new matrix type given the type and dependently-defined
/// dimensions.
QualType RebuildDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 929a895a3dbeb..a93f7ecedb3ec 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7583,8 +7583,7 @@ void TypeLocReader::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
TL.setAttrColumnOperand(Reader.readExpr());
}
-void TypeLocReader::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocReader::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
TL.setAttrNameLoc(readSourceLocation());
TL.setAttrOperandParensRange(readSourceRange());
TL.setAttrScopeOperand(Reader.readExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 794bba58afd06..daa5d0dcde752 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -452,8 +452,7 @@ void TypeLocWriter::VisitConstantMatrixTypeLoc(ConstantMatrixTypeLoc TL) {
Record.AddStmt(TL.getAttrColumnOperand());
}
-void TypeLocWriter::VisitCooperativeMatrixTypeLoc(
- CooperativeMatrixTypeLoc TL) {
+void TypeLocWriter::VisitCooperativeMatrixTypeLoc(CooperativeMatrixTypeLoc TL) {
addSourceLocation(TL.getAttrNameLoc());
SourceRange range = TL.getAttrOperandParensRange();
addSourceLocation(range.getBegin());
>From bd6c8382e21e25403eb7b7b4b2fa3f297b05d2b5 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 15 Jul 2026 12:12:32 -0400
Subject: [PATCH 08/12] Add tests for coop_mat_init and coop_mat_length
---
.../coop-mat-init-length-codegen.cl | 112 ++++++++++++++++++
.../SemaOpenCL/coop-mat-init-length-sema.cl | 94 +++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
create mode 100644 clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
new file mode 100644
index 0000000000000..b18f5617338c8
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -0,0 +1,112 @@
+// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
+// RUN: | FileCheck %s
+
+// ---------------------------------------------------------------------------
+// Type definitions.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init — float scalar.
+// Expected IR: call __spirv_CompositeConstruct with the scalar value,
+// returning a spirv.CooperativeMatrixKHR TargetExtType.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_float
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float
+
+kernel void test_init_float(global float *out) {
+ MatA_t a;
+ a = coop_mat_init(1.0f);
+ (void)a;
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init — integer scalar.
+// Expected IR: call __spirv_CompositeConstruct with an i32 value.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_int
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(i32
+
+kernel void test_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+ (void)m;
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init — zero initialisation.
+// Expected IR: call __spirv_CompositeConstruct with 0.0.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_zero
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+
+kernel void test_init_zero(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ (void)acc;
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length — returns unsigned int.
+// Expected IR: call __spirv_CooperativeMatrixLengthKHR, result stored into
+// an i32 alloca.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
+
+kernel void test_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ *out = len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_init then coop_mat_length — combined flow.
+// Verifies the TargetExtType produced by init flows into length unchanged.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_init_then_length
+// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
+// CHECK-SAME: @__spirv_CompositeConstruct(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_init_then_length(global unsigned int *out) {
+ MatA_t a;
+ a = coop_mat_init(2.0f);
+ *out = coop_mat_length(a);
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length — calling convention is SPIR_FUNC.
+// The call must NOT be a plain 'call' — it must be 'call spir_func'.
+// ---------------------------------------------------------------------------
+
+// CHECK-LABEL: @{{.*}}test_length_cc
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+
+kernel void test_length_cc(global unsigned int *out) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ *out = coop_mat_length(acc);
+}
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
new file mode 100644
index 0000000000000..ab499bc1b4af7
--- /dev/null
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -0,0 +1,94 @@
+// Tests for coop_mat_init and coop_mat_length — Sema (positive).
+//
+// coop_mat_init and coop_mat_length have no entries in CheckBuiltinFunctionCall,
+// so arg-count and arg-type validation are not performed by Sema for these two
+// builtins. Only the positive (valid-code) path and the one assignment-target
+// diagnostic (err_coop_matrix_assignment, checked via AddInitializerToDecl) are
+// tested here.
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -fsyntax-only -verify %s
+
+// expected-no-diagnostics (sections 1-6 must compile cleanly)
+
+// ---------------------------------------------------------------------------
+// Type definitions shared across all tests.
+// ---------------------------------------------------------------------------
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_B))) MatB_t;
+typedef float __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_ACCUMULATOR))) MatC_t;
+typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
+ 16, 16,
+ CLK_COOPERATIVE_MATRIX_A))) MatA_int_t;
+
+// ---------------------------------------------------------------------------
+// 1. coop_mat_init: initialise each use role with a float scalar.
+// Two-step (declare then assign) is required because the return type is
+// fixed up in AddInitializerToDecl / CreateBuiltinBinOp.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_basic(void) {
+ MatA_t a;
+ MatB_t b;
+ MatC_t c;
+ a = coop_mat_init(1.0f);
+ b = coop_mat_init(2.0f);
+ c = coop_mat_init(0.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 2. coop_mat_init: integer element type.
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_int(void) {
+ MatA_int_t m;
+ m = coop_mat_init(42);
+}
+
+// ---------------------------------------------------------------------------
+// 3. coop_mat_init: re-initialise the same variable (chained assigns).
+// ---------------------------------------------------------------------------
+void test_coop_mat_init_chained(void) {
+ MatC_t acc;
+ acc = coop_mat_init(0.0f);
+ acc = coop_mat_init(1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// 4. coop_mat_length: basic usage — returns unsigned int directly.
+// No two-step required for the length call itself.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_basic(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int len = coop_mat_length(a);
+ (void)len;
+}
+
+// ---------------------------------------------------------------------------
+// 5. coop_mat_length: all three use roles.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_roles(void) {
+ MatA_t a; a = coop_mat_init(0.0f);
+ MatB_t b; b = coop_mat_init(0.0f);
+ MatC_t c; c = coop_mat_init(0.0f);
+ unsigned int la = coop_mat_length(a);
+ unsigned int lb = coop_mat_length(b);
+ unsigned int lc = coop_mat_length(c);
+ (void)la; (void)lb; (void)lc;
+}
+
+// ---------------------------------------------------------------------------
+// 6. coop_mat_length: result used in arithmetic.
+// ---------------------------------------------------------------------------
+void test_coop_mat_length_arith(void) {
+ MatA_t a;
+ a = coop_mat_init(0.0f);
+ unsigned int half_len = coop_mat_length(a) / 2u;
+ (void)half_len;
+}
>From 1981cbd8d6a62533345dd848f0a9e38d003e6f10 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Wed, 29 Jul 2026 19:21:14 -0400
Subject: [PATCH 09/12] Add mangling support
Signed-off-by: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
---
clang/lib/CodeGen/CGBuiltin.cpp | 91 ++++++++-
.../coop-mat-codegen-mangling.cl | 174 ++++++++++++++++++
.../coop-mat-init-length-codegen.cl | 14 +-
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 15 ++
4 files changed, 280 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index fa48fc0eb2458..ddc5c31d75f22 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2988,6 +2988,62 @@ static llvm::TargetExtType *getTargetExtType(CodeGenFunction &CGF,
return RetType;
}
+/// Returns a suffix encoding ALL parameters of a spirv.CooperativeMatrixKHR
+/// TargetExtType: element type + scope + rows + cols + use.
+/// E.g. target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) ->
+/// "_f32_sc3_16x16_u2"
+static std::string getCoopMatFullSuffix(llvm::Type *Ty) {
+ auto *TET = cast<llvm::TargetExtType>(Ty);
+ llvm::Type *ElemTy = TET->getTypeParameter(0);
+
+ // Element type string
+ std::string ElemStr;
+ if (ElemTy->isFloatTy())
+ ElemStr = "f32";
+ else if (ElemTy->isHalfTy())
+ ElemStr = "f16";
+ else if (ElemTy->isDoubleTy())
+ ElemStr = "f64";
+ else if (auto *IntTy = dyn_cast<llvm::IntegerType>(ElemTy))
+ ElemStr = "i" + std::to_string(IntTy->getBitWidth());
+ else
+ llvm_unreachable("Unsupported cooperative matrix element type");
+
+ // Integer parameters: scope, rows, cols, use
+ unsigned Scope = TET->getIntParameter(0);
+ unsigned Rows = TET->getIntParameter(1);
+ unsigned Cols = TET->getIntParameter(2);
+ unsigned Use = TET->getIntParameter(3);
+
+ return "_" + ElemStr + "_sc" + std::to_string(Scope) + "_" +
+ std::to_string(Rows) + "x" + std::to_string(Cols) + "_u" +
+ std::to_string(Use);
+}
+
+/// Returns a name suffix encoding the address space of a pointer type,
+/// matching OpenCL/SPIR-V address space conventions.
+/// E.g. ptr addrspace(1) -> "_global"
+/// ptr addrspace(3) -> "_local"
+/// ptr addrspace(0) -> "_private"
+/// ptr addrspace(4) -> "_generic"
+static std::string getPtrAddrSpaceSuffix(llvm::Type *PtrTy) {
+ unsigned AS = cast<llvm::PointerType>(PtrTy)->getAddressSpace();
+ switch (AS) {
+ case 0:
+ return "_private";
+ case 1:
+ return "_global";
+ case 2:
+ return "_constant";
+ case 3:
+ return "_local";
+ case 4:
+ return "_generic";
+ default:
+ return "_as" + std::to_string(AS);
+ }
+}
+
} // namespace
RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
@@ -4866,7 +4922,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy = llvm::FunctionType::get(
RetType, {Ptr->getType(), Layout->getType(), Stride->getType()}, false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee LoadFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4899,7 +4958,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
VoidTy, {Ptr->getType(), ArgType, Layout->getType(), Stride->getType()},
false);
// Function name mangling.
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getPtrAddrSpaceSuffix(ConvertType(E->getArg(0)->getType())) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(1)->getType()));
llvm::FunctionCallee StoreFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
@@ -4941,7 +5003,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, CType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(Arg0->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg1->getType())) +
+ getCoopMatFullSuffix(ConvertType(Arg2->getType()));
llvm::FunctionCallee MatMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(MatMulFn.getCallee()))
@@ -4981,6 +5047,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
(ElTy->isIntegerTy())
? getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel)
: getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel, /*IsFloat*/ true);
+ // Function name mangling.
+ Name = Name + getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee BinaryFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(BinaryFn.getCallee()))
@@ -5010,7 +5078,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, BType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarMulFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarMulFn.getCallee()))
@@ -5038,7 +5108,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {AType, Int1Ty}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee ScalarNegFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(ScalarNegFn.getCallee()))
@@ -5055,7 +5127,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
// Set function type.
llvm::FunctionType *FTy =
llvm::FunctionType::get(RetType, {Init->getType()}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getType()));
llvm::FunctionCallee InitFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(InitFn.getCallee()))
@@ -5072,7 +5146,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::FunctionType *FTy =
llvm::FunctionType::get(ConvertType(E->getType()), {AType}, false);
- std::string Name = getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel);
+ // Function name mangling.
+ std::string Name =
+ getSPIRVBuiltinName(BuiltinIDIfNoAsmLabel) +
+ getCoopMatFullSuffix(ConvertType(E->getArg(0)->getType()));
llvm::FunctionCallee LengthFn =
CGM.getModule().getOrInsertFunction(Name, FTy);
if (auto *F = llvm::dyn_cast<llvm::Function>(LengthFn.getCallee()))
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
new file mode 100644
index 0000000000000..9fec09287f890
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -0,0 +1,174 @@
+// clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+//
+// Tests for function-name mangling of SPIR-V cooperative matrix intrinsics.
+// Covers three mangling axes:
+// (A) coop_mat_mulAdd -- same element type, different matrix dimensions / scope / use
+// (B) coop_mat_mulAdd -- different element types (float vs int)
+// (C) coop_mat_load -- different pointer address spaces
+// (D) coop_mat_store -- different pointer address spaces
+//
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
+// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
+// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
+
+#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
+#define USE_A CLK_COOPERATIVE_MATRIX_A
+#define USE_B CLK_COOPERATIVE_MATRIX_B
+#define USE_C CLK_COOPERATIVE_MATRIX_ACCUMULATOR
+#define ROW_MAJOR CLK_COOPERATIVE_MATRIX_LAYOUT_ROW_MAJOR
+
+// ---------------------------------------------------------------------------
+// Matrix type aliases used across tests
+// ---------------------------------------------------------------------------
+
+// --- float 16x16 (MulAdd group 1) ---
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatF_16x16_A;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatF_16x16_B;
+typedef float __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatF_16x16_C;
+
+// --- float 8x32 / 32x8 / 8x8 (MulAdd group 2 -- same elem type, diff dims) ---
+typedef float __attribute__((coop_mat(SCOPE, 8, 32, USE_A))) MatF_8x32_A;
+typedef float __attribute__((coop_mat(SCOPE, 32, 8, USE_B))) MatF_32x8_B;
+typedef float __attribute__((coop_mat(SCOPE, 8, 8, USE_C))) MatF_8x8_C;
+
+// --- int 16x16 (MulAdd group 3 -- different element type to group 1) ---
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_A))) MatI_16x16_A;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_B))) MatI_16x16_B;
+typedef int __attribute__((coop_mat(SCOPE, 16, 16, USE_C))) MatI_16x16_C;
+
+// ===========================================================================
+// (A) MulAdd: same element type (float), different matrix dimensions
+// 16x16 x 16x16 -> 16x16 vs 8x32 x 32x8 -> 8x8
+// Must produce TWO distinct __spirv_CooperativeMatrixMulAddKHR_* symbols.
+// ===========================================================================
+
+kernel void test_muladd_f32_16x16(__global float *ptr) {
+ MatF_16x16_A a;
+ MatF_16x16_B b;
+ MatF_16x16_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ b = coop_mat_load(ptr, ROW_MAJOR, 16);
+ c = coop_mat_load(ptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+kernel void test_muladd_f32_8x32(__global float *ptr) {
+ MatF_8x32_A a;
+ MatF_32x8_B b;
+ MatF_8x8_C c, r;
+ a = coop_mat_load(ptr, ROW_MAJOR, 8);
+ b = coop_mat_load(ptr, ROW_MAJOR, 32);
+ c = coop_mat_load(ptr, ROW_MAJOR, 8);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_f32_8x32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x32_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_32x8_u{{[0-9]+}}
+// CHECK-SAME: _f32_sc{{[0-9]+}}_8x8_u{{[0-9]+}}
+
+// ===========================================================================
+// (B) MulAdd: different element types -- float 16x16 vs int 16x16
+// Same dimensions, different elem type -> must produce two distinct symbols.
+// ===========================================================================
+
+kernel void test_muladd_i32_16x16(__global int *iptr) {
+ MatI_16x16_A a;
+ MatI_16x16_B b;
+ MatI_16x16_C c, r;
+ a = coop_mat_load(iptr, ROW_MAJOR, 16);
+ b = coop_mat_load(iptr, ROW_MAJOR, 16);
+ c = coop_mat_load(iptr, ROW_MAJOR, 16);
+ r = coop_mat_mulAdd(a, b, c);
+ (void)r;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_muladd_i32_16x16
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixMulAddKHR
+// CHECK-SAME: _i32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+
+// ===========================================================================
+// (C) Load: different pointer address spaces
+// __global (addrspace 1) vs __local (addrspace 3) vs private (addrspace 0)
+// Must produce THREE distinct __spirv_CooperativeMatrixLoadKHR_* symbols.
+// ===========================================================================
+
+kernel void test_load_global(__global float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_load_local(__local float *ptr) {
+ MatF_16x16_A a;
+ a = coop_mat_load(ptr, ROW_MAJOR, 16);
+ (void)a;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_load_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (D) Store: different pointer address spaces
+// Same matrix type, __global vs __local -> two distinct symbols.
+// ===========================================================================
+
+kernel void test_store_global(__global float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_global
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(1)
+
+kernel void test_store_local(__local float *ptr, MatF_16x16_A a) {
+ coop_mat_store(ptr, a, ROW_MAJOR, 16);
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_store_local
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc{{[0-9]+}}_16x16_u{{[0-9]+}}
+// CHECK-SAME: ptr addrspace(3)
+
+// ===========================================================================
+// (E) Negative: confirm NO cross-contamination between address space variants
+// i.e. the _global load function is NOT called for the _local case.
+// ===========================================================================
+
+kernel void test_no_cross_contamination(__global float *gptr,
+ __local float *lptr) {
+ MatF_16x16_A ag, al;
+ ag = coop_mat_load(gptr, ROW_MAJOR, 16);
+ al = coop_mat_load(lptr, ROW_MAJOR, 16);
+ (void)ag; (void)al;
+}
+// CHECK-LABEL: @__clang_ocl_kern_imp_test_no_cross_contamination
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32
+// CHECK: call {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_global_f32_{{.*}}addrspace(3)
+// CHECK-NOT: @__spirv_CooperativeMatrixLoadKHR_local_f32_{{.*}}addrspace(1)
+
+// Verify distinct declare lines exist (these appear at end of IR).
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_16x16_u0_f32_sc3_16x16_u1_f32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x32_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_32x8_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_f32_sc3_8x32_u0_f32_sc3_32x8_u1_f32_sc3_8x8_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u1
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_global_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixMulAddKHR_i32_sc3_16x16_u0_i32_sc3_16x16_u1_i32_sc3_16x16_u2
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixLoadKHR_local_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_global_f32_sc3_16x16_u0
+// CHECK-DECL: declare {{.*}} @__spirv_CooperativeMatrixStoreKHR_local_f32_sc3_16x16_u0
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index b18f5617338c8..8dd1aab2b2a13 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -26,7 +26,7 @@ typedef int __attribute__((coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP,
// CHECK-LABEL: @{{.*}}test_init_float
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_float(global float *out) {
MatA_t a;
@@ -41,7 +41,7 @@ kernel void test_init_float(global float *out) {
// CHECK-LABEL: @{{.*}}test_init_int
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(i32
+// CHECK-SAME: @__spirv_CompositeConstruct
kernel void test_init_int(void) {
MatA_int_t m;
@@ -56,7 +56,7 @@ kernel void test_init_int(void) {
// CHECK-LABEL: @{{.*}}test_init_zero
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(float {{.*}}0
+// CHECK-SAME: @__spirv_CompositeConstruct_{{.*}}(float {{.*}}0
kernel void test_init_zero(void) {
MatC_t acc;
@@ -71,7 +71,7 @@ kernel void test_init_zero(void) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
// CHECK-SAME: target("spirv.CooperativeMatrixKHR"
kernel void test_length(global unsigned int *out) {
@@ -88,8 +88,8 @@ kernel void test_length(global unsigned int *out) {
// CHECK-LABEL: @{{.*}}test_init_then_length
// CHECK: call spir_func target("spirv.CooperativeMatrixKHR"
-// CHECK-SAME: @__spirv_CompositeConstruct(
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK-SAME: @__spirv_CompositeConstruct
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_init_then_length(global unsigned int *out) {
MatA_t a;
@@ -103,7 +103,7 @@ kernel void test_init_then_length(global unsigned int *out) {
// ---------------------------------------------------------------------------
// CHECK-LABEL: @{{.*}}test_length_cc
-// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR(
+// CHECK: call spir_func i32 @__spirv_CooperativeMatrixLengthKHR
kernel void test_length_cc(global unsigned int *out) {
MatC_t acc;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 236da61192f1a..4292a74bd63b3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -552,6 +552,21 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
}
std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
+ // Cooperative-matrix builtins emitted by Clang CodeGen carry mangling
+ // suffixes to avoid IR function-signature collisions, e.g.:
+ // "__spirv_CooperativeMatrixLoadKHR_f32_sc3_16x16_u0_global"
+ // Strip everything from the first '_' after "__spirv_CooperativeMatrix"
+ // so the name matches the canonical DemangledNativeBuiltin entry in
+ // SPIRVBuiltins.td (e.g. "__spirv_CooperativeMatrixLoadKHR").
+ static constexpr StringRef CoopMatPrefix = "__spirv_CooperativeMatrix";
+ if (Name.starts_with(CoopMatPrefix)) {
+ size_t SuffixPos = Name.find('_', CoopMatPrefix.size());
+ if (SuffixPos != StringRef::npos)
+ return Name.substr(0, SuffixPos).str();
+ // No mangling suffix present — return the name as-is.
+ return Name.str();
+ }
+
bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
>From 290e0778ad57e002f68dfd9b0174bd35764774b1 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 13:23:45 -0400
Subject: [PATCH 10/12] Change name of extension to cl_khr_cooperative_matrix
---
clang/include/clang/Basic/Builtins.td | 2 +-
clang/include/clang/Basic/OpenCLExtensions.def | 2 +-
.../test/CodeGenOpenCL/coop-mat-codegen-mangling.cl | 2 +-
clang/test/CodeGenOpenCL/coop-mat-codegen.cl | 2 +-
.../CodeGenOpenCL/coop-mat-init-length-codegen.cl | 2 +-
clang/test/Preprocessor/coop-mat-opencl-ext.cl | 12 ++++++------
clang/test/SemaOpenCL/coop-mat-ast-utils.cl | 8 ++++----
clang/test/SemaOpenCL/coop-mat-init-length-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema-neg.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-sema.cl | 2 +-
clang/test/SemaOpenCL/coop-mat-type-infra.cl | 6 +++---
11 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 6b19fea681489..8371131f22e59 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1931,7 +1931,7 @@ def MatrixColumnMajorStore : Builtin {
let Prototype = "void(...)";
}
-// Cooperative Matrix builtins for cl_ext_kernel_cooperative_matrix
+// Cooperative Matrix builtins for cl_khr_cooperative_matrix
def CoopMatLoad : Builtin {
let Spellings = ["coop_mat_load"];
let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 89e96dca85a59..8a583ad504c70 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -71,7 +71,7 @@ OPENCL_EXTENSION(cl_khr_int64_extended_atomics, true, 100)
OPENCL_EXTENSION(cl_khr_depth_images, true, 100)
OPENCL_COREFEATURE(cl_khr_extended_bit_ops, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_ext_float_atomics, false, 100)
-OPENCL_EXTENSION(cl_ext_kernel_cooperative_matrix, true, 100)
+OPENCL_EXTENSION(cl_khr_cooperative_matrix, true, 100)
OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, true, 100)
OPENCL_COREFEATURE(cl_khr_integer_dot_product, false, 100, OCL_C_31)
OPENCL_EXTENSION(cl_khr_kernel_clock, false, 100)
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
index 9fec09287f890..9d55bd8c61cc3 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen-mangling.cl
@@ -8,7 +8,7 @@
// (D) coop_mat_store -- different pointer address spaces
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o %t.ll %s
// RUN: FileCheck --check-prefix=CHECK %s < %t.ll
// RUN: FileCheck --check-prefix=CHECK-DECL %s < %t.ll
diff --git a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
index 58b7a882fdc87..ad8df2939269f 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-codegen.cl
@@ -4,7 +4,7 @@
// SPIR-V intrinsic names.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
index 8dd1aab2b2a13..dfa763b060e72 100644
--- a/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
+++ b/clang/test/CodeGenOpenCL/coop-mat-init-length-codegen.cl
@@ -1,7 +1,7 @@
// Tests for coop_mat_init and coop_mat_length — CodeGen (IR shape).
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -O0 -o - %s \
// RUN: | FileCheck %s
diff --git a/clang/test/Preprocessor/coop-mat-opencl-ext.cl b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
index 7181efbe7e2a8..f77fe542fd210 100644
--- a/clang/test/Preprocessor/coop-mat-opencl-ext.cl
+++ b/clang/test/Preprocessor/coop-mat-opencl-ext.cl
@@ -1,6 +1,6 @@
// clang/test/Preprocessor/coop_mat_opencl_ext.cl
//
-// Patch 2: cl_ext_kernel_cooperative_matrix registration in
+// Patch 2: cl_khr_cooperative_matrix registration in
// OpenCLExtensions.def and enum definitions in opencl-c-base.h.
//
// Tests: extension macro is predefined when enabled, extension can be
@@ -9,23 +9,23 @@
// ── 2a. Extension macro is predefined when the extension is enabled ─────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E -dM %s \
// RUN: | FileCheck %s --check-prefix=EXT
-// EXT: cl_ext_kernel_cooperative_matrix
+// EXT: cl_khr_cooperative_matrix
// ── 2b. Extension is NOT predefined when explicitly disabled ────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=-cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=-cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -E %s \
// RUN: | FileCheck %s --check-prefix=NOEXT
-// NOEXT-NOT: cl_ext_kernel_cooperative_matrix
+// NOEXT-NOT: cl_khr_cooperative_matrix
// ── 2c. Enum constants are visible when extension is enabled ─────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
index c6230eb94531f..e8fad815806ec 100644
--- a/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
+++ b/clang/test/SemaOpenCL/coop-mat-ast-utils.cl
@@ -6,24 +6,24 @@
//
// ── 3a. Itanium mangling ────────────────────────────────────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefix=MANGLE
//
// ── 3b. Serialisation round-trip (TypeLoc reader/writer) ────────────────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "void pch_probe(MatA_t a);" > %t.pch_probe.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch \
// RUN: -ast-dump %t.pch_probe.cl \
// RUN: | FileCheck %s --check-prefix=PCH
//
// ── 3c. Structural equivalence (no diagnostics on compatible pair) ───────────
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
index ab499bc1b4af7..799f7a9e15a35 100644
--- a/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-init-length-sema.cl
@@ -7,7 +7,7 @@
// tested here.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics (sections 1-6 must compile cleanly)
diff --git a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
index f8080448b6df5..0c27978d492a9 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema-neg.cl
@@ -3,7 +3,7 @@
// Patch 4: Sema — negative tests for diagnostic paths.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
#define SCOPE CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP
diff --git a/clang/test/SemaOpenCL/coop-mat-sema.cl b/clang/test/SemaOpenCL/coop-mat-sema.cl
index cec1f278b0037..bcedd94fdbf17 100644
--- a/clang/test/SemaOpenCL/coop-mat-sema.cl
+++ b/clang/test/SemaOpenCL/coop-mat-sema.cl
@@ -5,7 +5,7 @@
//
// Valid code -- expected-no-diagnostics
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -fsyntax-only -verify %s
// expected-no-diagnostics
diff --git a/clang/test/SemaOpenCL/coop-mat-type-infra.cl b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
index 2e6700973e611..43adeebd94500 100644
--- a/clang/test/SemaOpenCL/coop-mat-type-infra.cl
+++ b/clang/test/SemaOpenCL/coop-mat-type-infra.cl
@@ -8,16 +8,16 @@
// TypeLoc operand slots, sizeof / getTypeInfoImpl.
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -ast-dump %s \
// RUN: | FileCheck %s --check-prefix=AST
//
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -emit-pch -o %t.pch %s
// RUN: echo "MatA_float16x16 g;" > %t.aux.cl
// RUN: %clang_cc1 -triple spirv64-unknown-unknown \
-// RUN: -cl-std=CL2.0 -cl-ext=+cl_ext_kernel_cooperative_matrix \
+// RUN: -cl-std=CL2.0 -cl-ext=+cl_khr_cooperative_matrix \
// RUN: -finclude-default-header -O0 -include-pch %t.pch -ast-dump %t.aux.cl \
// RUN: | FileCheck %s --check-prefix=PCH
>From 640a1ac6de6acd451c557ad78aca6c3757ec35e8 Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Tue, 4 Aug 2026 14:37:44 -0400
Subject: [PATCH 11/12] Fix test fails
---
clang/lib/Sema/SemaType.cpp | 6 ++++++
clang/test/AST/undocumented-attrs.cpp | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 00579d7136825..055f1e7bc18d1 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -8995,6 +8995,12 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr,
Scope, Use, /* IsCoopMat */ true);
if (!T.isNull())
CurType = T;
+ } else {
+ Expr *RowsExpr = Attr.getArgAsExpr(0);
+ Expr *ColsExpr = Attr.getArgAsExpr(1);
+ QualType T = S.BuildMatrixType(CurType, RowsExpr, ColsExpr, Attr.getLoc());
+ if (!T.isNull())
+ CurType = T;
}
}
diff --git a/clang/test/AST/undocumented-attrs.cpp b/clang/test/AST/undocumented-attrs.cpp
index 64c7d165915cc..ed36f897ebfa7 100644
--- a/clang/test/AST/undocumented-attrs.cpp
+++ b/clang/test/AST/undocumented-attrs.cpp
@@ -28,6 +28,7 @@ CHECK-NEXT: Common
CHECK-NEXT: Const
CHECK-NEXT: ConsumableAutoCast
CHECK-NEXT: ConsumableSetOnRead
+CHECK-NEXT: CoopMatrixType
CHECK-NEXT: FormatArg
CHECK-NEXT: GuardedBy
CHECK-NEXT: GuardedVar
@@ -89,4 +90,4 @@ CHECK-NEXT: Visibility
CHECK-NEXT: WeakImport
CHECK-NEXT: WeakRef
CHECK-NEXT: WorkGroupSizeHint
-CHECK-NEXT: Total: 83
+CHECK-NEXT: Total: 84
>From b7a4d3956bb8c0f56f94328c120de7137bec54dd Mon Sep 17 00:00:00 2001
From: Arvind Sudarsanam <asudarsa at qti.qualcomm.com>
Date: Sun, 9 Aug 2026 00:57:18 -0400
Subject: [PATCH 12/12] Fix failing lldb build and a couple of tests
---
.../test/Misc/pragma-attribute-supported-attributes-list.test | 1 +
lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 3 +++
llvm/test/Verifier/coop-mat-verifier-fix.ll | 4 ++--
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 8bca68e2119e7..c3803c29831e1 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -64,6 +64,7 @@
// CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
// CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
// CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CoopMatrixType (SubjectMatchRule_type_alias, SubjectMatchRule_variable, SubjectMatchRule_variable_is_parameter, SubjectMatchRule_field)
// CHECK-NEXT: CoroAwaitElidable (SubjectMatchRule_record)
// CHECK-NEXT: CoroAwaitElidableArgument (SubjectMatchRule_variable_is_parameter)
// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function)
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index b7646e783364e..42b2d6d21970c 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -4209,6 +4209,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
// Matrix types that we're not sure how to display at the moment.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5075,6 +5076,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type) {
break;
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
@@ -5248,6 +5250,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
// Matrix types we're not sure how to display yet.
case clang::Type::ConstantMatrix:
+ case clang::Type::CooperativeMatrix:
case clang::Type::DependentSizedMatrix:
break;
diff --git a/llvm/test/Verifier/coop-mat-verifier-fix.ll b/llvm/test/Verifier/coop-mat-verifier-fix.ll
index c4e206b76b760..7f2eedb657d41 100644
--- a/llvm/test/Verifier/coop-mat-verifier-fix.ll
+++ b/llvm/test/Verifier/coop-mat-verifier-fix.ll
@@ -22,14 +22,14 @@
declare void @llvm.experimental.noalias.scope.decl(metadata)
define void @test_no_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
; Third arg present and <= 64 bits wide — must also pass
define void @test_valid_third_arg() {
- %domain = call token @llvm.experimental.noalias.scope.decl(
+ call void @llvm.experimental.noalias.scope.decl(
metadata !0)
ret void
}
More information about the lldb-commits
mailing list