[llvm] [SPIR-V] Add SPV_INTEL_joint_matrix extension (PR #118578)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 3 18:20:10 PST 2024
https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/118578
The spec is available here:
https://github.com/intel/llvm/pull/12497
The PR doesn't add OpCooperativeMatrixApplyFunctionINTEL instruction as it's still experimental and not properly tested E2E.
The PR also fixes few bugs in the related code:
1. CooperativeMatrixMulAddKHR optional operand must be literal, not a constant;
2. Fixed available capabilities table creation for a case, when a single extension adds few capabilities, that occupy not contiguous op codes.
>From 0556328ee5adec96f38c2105536e33d852ab216f Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Tue, 3 Dec 2024 18:14:06 -0800
Subject: [PATCH] [SPIR-V] Add SPV_INTEL_joint_matrix extension
The spec is available here:
https://github.com/intel/llvm/pull/12497
The PR doesn't add OpCooperativeMatrixApplyFunctionINTEL instruction
as it's still experimental and not properly tested E2E.
The PR also fixes few bugs in the related code:
1. CooperativeMatrixMulAddKHR optional operand must be literal, not a
constant;
2. Fixed available capabilities table creation for a case, when a single
extension adds few capabilities, that occupy not contiguous op codes.
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
llvm/docs/SPIRVUsage.rst | 2 +
.../SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp | 8 +-
.../Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h | 10 ++
.../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 27 +++++
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 45 +++++++-
llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 7 ++
llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp | 2 +
llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 17 +++
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 104 ++++++++++++++++++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 68 ++++++++++++
.../cooperative_matrix_bf16.ll | 46 ++++++++
.../cooperative_matrix_checked.ll | 59 ++++++++++
.../cooperative_matrix_get_coord.ll | 35 ++++++
.../cooperative_matrix_packed.ll | 78 +++++++++++++
.../cooperative_matrix_prefetch.ll | 34 ++++++
.../cooperative_matrix_tf32.ll | 46 ++++++++
.../cooperative_matrix.ll | 2 +-
17 files changed, 582 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 28e919fdf516a0..8f7ac71f8026b3 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -179,6 +179,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
- Introduces two new storage classes that are subclasses of the CrossWorkgroup storage class that provides additional information that can enable optimization.
* - ``SPV_INTEL_variable_length_array``
- Allows to allocate local arrays whose number of elements is unknown at compile time.
+ * - ``SPV_INTEL_joint_matrix``
+ - Adds few matrix capabilities on top of SPV_KHR_cooperative_matrix extension, such as matrix prefetch, get element coordinate and checked load/store/construct instructions, tensor float 32 and bfloat type interpretations for multuply-add instruction.
* - ``SPV_KHR_bit_instructions``
- Enables bit instructions to be used by SPIR-V modules without requiring the Shader capability.
* - ``SPV_KHR_expect_assume``
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
index 0f9a2a69e07390..67bf4596152492 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
@@ -137,8 +137,12 @@ getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension) {
CapabilityList Capabilities;
while (Entry &&
- Entry->Category == SPIRV::OperandCategory::CapabilityOperand &&
- Entry->ReqExtension == Extension) {
+ Entry->Category == SPIRV::OperandCategory::CapabilityOperand) {
+ // Some capabilities' codes might go not in order.
+ if (Entry->ReqExtension != Extension) {
+ ++Entry;
+ continue;
+ }
Capabilities.push_back(
static_cast<SPIRV::Capability::Capability>(Entry->Value));
++Entry;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index 44625793e94138..823c33ecb6bd38 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -207,6 +207,16 @@ namespace Opcode {
#include "SPIRVGenTables.inc"
} // namespace Opcode
+namespace CooperativeMatrixLayout {
+#define GET_CooperativeMatrixLayout_DECL
+#include "SPIRVGenTables.inc"
+} // namespace Opcode
+
+namespace CooperativeMatrixOperands {
+#define GET_CooperativeMatrixOperands_DECL
+#include "SPIRVGenTables.inc"
+} // namespace Opcode
+
struct ExtendedBuiltin {
StringRef Name;
InstructionSet::InstructionSet Set;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index ff8759755e5176..d05a0e87ca870e 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -211,6 +211,33 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
// are part of the variable value.
printOpConstantVarOps(MI, NumFixedOps - 1, OS);
break;
+ case SPIRV::OpCooperativeMatrixMulAddKHR: {
+ const unsigned NumOps = MI->getNumOperands();
+ if (NumFixedOps == NumOps)
+ break;
+
+ OS << ' ';
+ const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm();
+ if (MulAddOp == 0) {
+ printSymbolicOperand<
+ OperandCategory::CooperativeMatrixOperandsOperand>(
+ MI, FirstVariableIndex, OS);
+ } else {
+ std::string Buffer;
+ for (unsigned Mask = 0x1;
+ Mask != SPIRV::CooperativeMatrixOperands::MatrixResultBFloat16ComponentsINTEL;
+ Mask <<= 1) {
+ if (MulAddOp & Mask) {
+ if (!Buffer.empty())
+ Buffer += '|';
+ Buffer += getSymbolicOperandMnemonic(
+ OperandCategory::CooperativeMatrixOperandsOperand, Mask);
+ }
+ }
+ OS << Buffer;
+ }
+ break;
+ }
default:
printRemainingVariableOps(MI, NumFixedOps, OS);
break;
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 45a49674d4ca21..28fcbda2d1df89 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1969,15 +1969,50 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call,
const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
unsigned Opcode =
SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
- bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR;
+ bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&
+ Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&
+ Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
unsigned ArgSz = Call->Arguments.size();
unsigned LiteralIdx = 0;
- if (Opcode == SPIRV::OpCooperativeMatrixLoadKHR && ArgSz > 3)
- LiteralIdx = 3;
- else if (Opcode == SPIRV::OpCooperativeMatrixStoreKHR && ArgSz > 4)
- LiteralIdx = 4;
+ switch(Opcode) {
+ // Memory operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixLoadKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreKHR:
+ LiteralIdx = ArgSz > 4 ? 4 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ LiteralIdx = ArgSz > 7 ? 7 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ LiteralIdx = ArgSz > 8 ? 8 : 0;
+ break;
+ // Cooperative Matrix Operands operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixMulAddKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
+ };
+
SmallVector<uint32_t, 1> ImmArgs;
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+ const uint32_t CacheLevel =
+ getConstFromIntrinsic(Call->Arguments[3], MRI);
+ auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)
+ .addUse(Call->Arguments[0]) // pointer
+ .addUse(Call->Arguments[1]) // rows
+ .addUse(Call->Arguments[2]) // columns
+ .addImm(CacheLevel) // cache level
+ .addUse(Call->Arguments[4]); // memory layout
+ if (ArgSz > 5)
+ MIB.addUse(Call->Arguments[5]); // stride
+ if (ArgSz > 6) {
+ const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI);
+ MIB.addImm(MemOp); // memory operand
+ }
+ return true;
+ }
if (LiteralIdx > 0)
ImmArgs.push_back(getConstFromIntrinsic(Call->Arguments[LiteralIdx], MRI));
Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index dc2da4a3a5647a..e29013d28aafe4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -695,6 +695,13 @@ defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreKHR", OpenCL_std, C
defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixMulAddKHR", OpenCL_std, CoopMatr, 3, 4, OpCooperativeMatrixMulAddKHR>;
defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLengthKHR", OpenCL_std, CoopMatr, 1, 1, OpCooperativeMatrixLengthKHR>;
+// Cooperative Matrix Intel builtin records:
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixPrefetchINTEL", OpenCL_std, CoopMatr, 5, 7, OpCooperativeMatrixPrefetchINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadCheckedINTEL", OpenCL_std, CoopMatr, 6, 8, OpCooperativeMatrixLoadCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreCheckedINTEL", OpenCL_std, CoopMatr, 7, 9, OpCooperativeMatrixStoreCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixConstructCheckedINTEL", OpenCL_std, CoopMatr, 5, 5, OpCooperativeMatrixConstructCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixGetElementCoordINTEL", OpenCL_std, CoopMatr, 2, 2, OpCooperativeMatrixGetElementCoordINTEL>;
+
//===----------------------------------------------------------------------===//
// Class defining a work/sub group builtin that should be translated into a
// SPIR-V instruction using the defined properties.
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index e78fc5ce18707b..fb05c1fdbd1e3b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -51,6 +51,8 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
SPIRV::Extension::Extension::SPV_INTEL_subgroups},
{"SPV_INTEL_media_block_io",
SPIRV::Extension::Extension::SPV_INTEL_media_block_io},
+ {"SPV_INTEL_joint_matrix",
+ SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
{"SPV_KHR_uniform_group_instructions",
SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions},
{"SPV_KHR_no_integer_wrap_decoration",
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 53f1b644a94983..d95803fea56a58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -895,6 +895,23 @@ def OpCooperativeMatrixMulAddKHR: Op<4459, (outs ID:$res),
def OpCooperativeMatrixLengthKHR: Op<4460, (outs ID:$res), (ins TYPE:$type, ID:$coop_matr_type),
"$res = OpCooperativeMatrixLengthKHR $type $coop_matr_type">;
+// SPV_INTEL_joint_matrix
+def OpCooperativeMatrixLoadCheckedINTEL: Op<6193, (outs ID:$res),
+ (ins TYPE:$resType, ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+ "$res = OpCooperativeMatrixLoadCheckedINTEL $resType $pointer $xOffset $yOffset $memory_layout $height $width">;
+def OpCooperativeMatrixStoreCheckedINTEL: Op<6194, (outs),
+ (ins ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$objectToStore, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+ "OpCooperativeMatrixStoreCheckedINTEL $pointer $xOffset $yOffset $objectToStore $memory_layout $height $width">;
+def OpCooperativeMatrixConstructCheckedINTEL: Op<6195, (outs ID:$res),
+ (ins TYPE:$resType, ID:$xOffset, ID:$yOffset, ID:$height, ID:$width, ID:$value),
+ "$res = OpCooperativeMatrixConstructCheckedINTEL $resType $xOffset $yOffset $height $width $value">;
+def OpCooperativeMatrixGetElementCoordINTEL: Op<6440, (outs ID:$res),
+ (ins TYPE:$resType, ID:$matrix, ID:$index),
+ "$res = OpCooperativeMatrixGetElementCoordINTEL $resType $matrix $index">;
+def OpCooperativeMatrixPrefetchINTEL: Op<6449, (outs),
+ (ins ID:$pointer, ID:$rows, ID:$columns, i32imm:$cacheLevel, ID:$memory_layout, variable_ops),
+ "OpCooperativeMatrixPrefetchINTEL $pointer $rows $columns $cacheLevel $memory_layout">;
+
// SPV_EXT_arithmetic_fence
def OpArithmeticFenceEXT: Op<6145, (outs ID:$res), (ins TYPE:$type, ID:$target),
"$res = OpArithmeticFenceEXT $type $target">;
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 20540814763157..ee7c08d324bb0c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1437,6 +1437,110 @@ void addInstrRequirements(const MachineInstr &MI,
Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
}
break;
+ case SPIRV::OpCooperativeMatrixMulAddKHR: {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+ report_fatal_error("Cooperative matrix instructions require the "
+ "following SPIR-V extension: "
+ "SPV_KHR_cooperative_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+ constexpr unsigned MulAddMaxSize = 6;
+ if (MI.getNumOperands() != MulAddMaxSize)
+ break;
+ const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
+ if (CoopOperands &
+ SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
+ }
+ if (CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixAAndBBFloat16ComponentsINTEL ||
+ CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixCBFloat16ComponentsINTEL ||
+ CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixResultBFloat16ComponentsINTEL) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
+ }
+ break;
+ }
+ case SPIRV::OpCooperativeMatrixLoadKHR:
+ case SPIRV::OpCooperativeMatrixStoreKHR:
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+ report_fatal_error("Cooperative matrix instructions require the "
+ "following SPIR-V extension: "
+ "SPV_KHR_cooperative_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+
+ // Check Layout operand in case if it's not a standart one and add the
+ // appropriate capability.
+ std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
+ {SPIRV::OpCooperativeMatrixLoadKHR, 3},
+ {SPIRV::OpCooperativeMatrixStoreKHR, 2},
+ {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
+ {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
+ {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}
+ };
+
+ const auto OpCode = MI.getOpcode();
+ const unsigned LayoutNum = LayoutToInstMap[OpCode];
+ Register RegLayout = MI.getOperand(LayoutNum).getReg();
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
+ if (MILayout->getOpcode() == SPIRV::OpConstantI) {
+ const unsigned LayoutVal = MILayout->getOperand(2).getImm();
+ if (LayoutVal == static_cast<unsigned>(
+ SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("PackedINTEL layout require the following SPIR-V "
+ "extension: SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
+ }
+ }
+
+ // Nothing to do.
+ if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||
+ OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
+ break;
+
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL "
+ "instructions require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
+ break;
+ }
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+ break;
+ }
+ case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL"
+ " instructions require the following SPIR-V extension:"
+ " SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+ break;
+ case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "
+ "following SPIR-V extension: SPV_INTEL_joint_matrix",
+ false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
+ break;
case SPIRV::OpKill: {
Reqs.addCapability(SPIRV::Capability::Shader);
} break;
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index a3a88acdd6c6ae..745d1e1aec67aa 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -170,6 +170,8 @@ def GroupOperationOperand : OperandCategory;
def KernelEnqueueFlagsOperand : OperandCategory;
def KernelProfilingInfoOperand : OperandCategory;
def OpcodeOperand : OperandCategory;
+def CooperativeMatrixLayoutOperand : OperandCategory;
+def CooperativeMatrixOperandsOperand : OperandCategory;
//===----------------------------------------------------------------------===//
// Multiclass used to define Extesions enum values and at the same time
@@ -305,6 +307,7 @@ defm SPV_INTEL_global_variable_fpga_decorations : ExtensionOperand<110>;
defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>;
defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>;
defm SPV_EXT_optnone : ExtensionOperand<113>;
+defm SPV_INTEL_joint_matrix : ExtensionOperand<114>;
//===----------------------------------------------------------------------===//
// Multiclass used to define Capabilities enum values and at the same time
@@ -492,6 +495,12 @@ defm CacheControlsINTEL : CapabilityOperand<6441, 0, 0, [SPV_INTEL_cache_control
defm CooperativeMatrixKHR : CapabilityOperand<6022, 0, 0, [SPV_KHR_cooperative_matrix], []>;
defm ArithmeticFenceEXT : CapabilityOperand<6144, 0, 0, [SPV_EXT_arithmetic_fence], []>;
defm SplitBarrierINTEL : CapabilityOperand<6141, 0, 0, [SPV_INTEL_split_barrier], []>;
+defm CooperativeMatrixCheckedInstructionsINTEL : CapabilityOperand<6192, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixPrefetchINTEL : CapabilityOperand<6411, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm PackedCooperativeMatrixINTEL : CapabilityOperand<6434, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixInvocationInstructionsINTEL : CapabilityOperand<6435, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixTF32ComponentTypeINTEL : CapabilityOperand<6436, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixBFloat16ComponentTypeINTEL : CapabilityOperand<6437, 0, 0, [SPV_INTEL_joint_matrix], []>;
//===----------------------------------------------------------------------===//
// Multiclass used to define SourceLanguage enum values and at the same time
@@ -1649,3 +1658,62 @@ defm GenericCastToPtr : OpcodeOperand<122>;
defm Bitcast : OpcodeOperand<124>;
defm ConvertPtrToU : OpcodeOperand<117>;
defm ConvertUToPtr : OpcodeOperand<120>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Layout enum values and at the
+// same time SymbolicOperand entries extensions and capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixLayout : GenericEnum, Operand<i32> {
+ let FilterClass = "CooperativeMatrixLayout";
+ let NameField = "Name";
+ let ValueField = "Value";
+}
+
+class CooperativeMatrixLayout<string name, bits<32> value> {
+ string Name = name;
+ bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixLayoutOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+ def : CooperativeMatrixLayout<NAME, value>;
+ defm : SymbolicOperandWithRequirements<CooperativeMatrixLayoutOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm RowMajorKHR : CooperativeMatrixLayoutOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm ColumnMajorKHR : CooperativeMatrixLayoutOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm PackedINTEL : CooperativeMatrixLayoutOperand<0x2, [SPV_INTEL_joint_matrix], [PackedCooperativeMatrixINTEL]>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Operands enum values and at the
+// same time SymbolicOperand entries with string mnemonics, extensions and
+// capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixOperands : GenericEnum, Operand<i32> {
+ let FilterClass = "CooperativeMatrixOperands";
+ let NameField = "Name";
+ let ValueField = "Value";
+ let PrintMethod = !strconcat("printSymbolicOperand<OperandCategory::", FilterClass, "Operand>");
+}
+
+class CooperativeMatrixOperands<string name, bits<32> value> {
+ string Name = name;
+ bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+ def : CooperativeMatrixOperands<NAME, value>;
+ defm : SymbolicOperandWithRequirements<CooperativeMatrixOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm NoneKHR : CooperativeMatrixOperandsOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixASignedComponentsKHR : CooperativeMatrixOperandsOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixBSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x2, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixCSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x4, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixResultSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x8, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm SaturatingAccumulationKHR : CooperativeMatrixOperandsOperand<0x10, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixAAndBTF32ComponentsINTEL : CooperativeMatrixOperandsOperand<0x20, [SPV_INTEL_joint_matrix], [CooperativeMatrixTF32ComponentTypeINTEL]>;
+defm MatrixAAndBBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x40, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixCBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x80, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixResultBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x100, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
new file mode 100644
index 00000000000000..192d769b8a3ce2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
@@ -0,0 +1,46 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: CooperativeMatrixBFloat16ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) #2
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
new file mode 100644
index 00000000000000..73ddaad0dbad19
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -0,0 +1,59 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const12:]] = OpConstant %[[#Int32Ty]] 12
+; CHECK-DAG: %[[#Const48:]] = OpConstant %[[#Int32Ty]] 48
+; CHECK-DAG: %[[#Const0:]] = OpConstant %[[#Int32Ty]] 0
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32Ty]] 3
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32Ty]] 1
+; CHECK-DAG: %[[#MatTy1:]] = OpTypeCooperativeMatrixKHR %[[#Int32Ty]] %[[#Const3]] %[[#Const12]] %[[#Const12]] %[[#Const2]]
+; CHECK-DAG: %[[#MatTy2:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const3]] %[[#Const12]] %[[#Const48]] %[[#Const0]]
+; CHECK-DAG: %[[#MatTy3:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const2]] %[[#Const48]] %[[#Const12]] %[[#Const1]]
+; CHECK: OpCooperativeMatrixConstructCheckedINTEL %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy3]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #2
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #2
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
new file mode 100644
index 00000000000000..4af85beb401ddc
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
@@ -0,0 +1,35 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixInvocationInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#MatrixTy:]] = OpTypeCooperativeMatrixKHR
+
+; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]]
+; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]]
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+$_ZTSZZ23matrix_multiply_checked = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+ %coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx)
+ ret void
+}
+; Function Attrs: convergent
+declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef)
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
new file mode 100644
index 00000000000000..50461af93084e1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
@@ -0,0 +1,78 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability PackedCooperativeMatrixINTEL
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+$_ZTSZZ23matrix_multiply_checked = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) #1
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #1
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #1
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #1
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #1
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #1
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #1
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #1
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
new file mode 100644
index 00000000000000..e68a5a2383e85f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
@@ -0,0 +1,34 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpExtension "SPV_INTEL_joint_matrix"
+
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) local_unnamed_addr #0 comdat {
+entry:
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) #1
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) #1
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) #1
+ ret void
+}
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
new file mode 100644
index 00000000000000..4fafca457110fa
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
@@ -0,0 +1,46 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: OpCapability CooperativeMatrixTF32ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) #2
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
index 1c41c7331cda8a..e290c1eaeabad8 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
@@ -21,7 +21,7 @@
; CHECK: %[[#Load1:]] = OpCooperativeMatrixLoadKHR %[[#MatTy2]]
; CHECK: OpCooperativeMatrixLengthKHR %[[#Int32Ty]] %[[#MatTy2:]]
; CHECK: OpCooperativeMatrixLoadKHR %[[#MatTy3]]
-; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] %[[#]] %[[#]] %[[#]] MatrixCSignedComponentsKHR|MatrixResultSignedComponentsKHR
; CHECK: OpCooperativeMatrixStoreKHR
define spir_kernel void @matr_mult(ptr addrspace(1) align 1 %_arg_accA, ptr addrspace(1) align 1 %_arg_accB, ptr addrspace(1) align 4 %_arg_accC, i64 %_arg_N, i64 %_arg_K) {
More information about the llvm-commits
mailing list