[llvm] [SPIR-V] Add SPV_INTEL_joint_matrix extension (PR #118578)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 4 06:59:10 PST 2024
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/118578
>From 0556328ee5adec96f38c2105536e33d852ab216f Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Tue, 3 Dec 2024 18:14:06 -0800
Subject: [PATCH 1/5] [SPIR-V] Add SPV_INTEL_joint_matrix extension
The spec is available here:
https://github.com/intel/llvm/pull/12497
The PR doesn't add OpCooperativeMatrixApplyFunctionINTEL instruction
as it's still experimental and not properly tested E2E.
The PR also fixes few bugs in the related code:
1. CooperativeMatrixMulAddKHR optional operand must be literal, not a
constant;
2. Fixed available capabilities table creation for a case, when a single
extension adds few capabilities, that occupy not contiguous op codes.
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
llvm/docs/SPIRVUsage.rst | 2 +
.../SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp | 8 +-
.../Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h | 10 ++
.../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 27 +++++
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 45 +++++++-
llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 7 ++
llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp | 2 +
llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 17 +++
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 104 ++++++++++++++++++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 68 ++++++++++++
.../cooperative_matrix_bf16.ll | 46 ++++++++
.../cooperative_matrix_checked.ll | 59 ++++++++++
.../cooperative_matrix_get_coord.ll | 35 ++++++
.../cooperative_matrix_packed.ll | 78 +++++++++++++
.../cooperative_matrix_prefetch.ll | 34 ++++++
.../cooperative_matrix_tf32.ll | 46 ++++++++
.../cooperative_matrix.ll | 2 +-
17 files changed, 582 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 28e919fdf516a0..8f7ac71f8026b3 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -179,6 +179,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
- Introduces two new storage classes that are subclasses of the CrossWorkgroup storage class that provides additional information that can enable optimization.
* - ``SPV_INTEL_variable_length_array``
- Allows to allocate local arrays whose number of elements is unknown at compile time.
+ * - ``SPV_INTEL_joint_matrix``
+ - Adds few matrix capabilities on top of SPV_KHR_cooperative_matrix extension, such as matrix prefetch, get element coordinate and checked load/store/construct instructions, tensor float 32 and bfloat type interpretations for multuply-add instruction.
* - ``SPV_KHR_bit_instructions``
- Enables bit instructions to be used by SPIR-V modules without requiring the Shader capability.
* - ``SPV_KHR_expect_assume``
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
index 0f9a2a69e07390..67bf4596152492 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
@@ -137,8 +137,12 @@ getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension) {
CapabilityList Capabilities;
while (Entry &&
- Entry->Category == SPIRV::OperandCategory::CapabilityOperand &&
- Entry->ReqExtension == Extension) {
+ Entry->Category == SPIRV::OperandCategory::CapabilityOperand) {
+ // Some capabilities' codes might go not in order.
+ if (Entry->ReqExtension != Extension) {
+ ++Entry;
+ continue;
+ }
Capabilities.push_back(
static_cast<SPIRV::Capability::Capability>(Entry->Value));
++Entry;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index 44625793e94138..823c33ecb6bd38 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -207,6 +207,16 @@ namespace Opcode {
#include "SPIRVGenTables.inc"
} // namespace Opcode
+namespace CooperativeMatrixLayout {
+#define GET_CooperativeMatrixLayout_DECL
+#include "SPIRVGenTables.inc"
+} // namespace Opcode
+
+namespace CooperativeMatrixOperands {
+#define GET_CooperativeMatrixOperands_DECL
+#include "SPIRVGenTables.inc"
+} // namespace Opcode
+
struct ExtendedBuiltin {
StringRef Name;
InstructionSet::InstructionSet Set;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index ff8759755e5176..d05a0e87ca870e 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -211,6 +211,33 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
// are part of the variable value.
printOpConstantVarOps(MI, NumFixedOps - 1, OS);
break;
+ case SPIRV::OpCooperativeMatrixMulAddKHR: {
+ const unsigned NumOps = MI->getNumOperands();
+ if (NumFixedOps == NumOps)
+ break;
+
+ OS << ' ';
+ const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm();
+ if (MulAddOp == 0) {
+ printSymbolicOperand<
+ OperandCategory::CooperativeMatrixOperandsOperand>(
+ MI, FirstVariableIndex, OS);
+ } else {
+ std::string Buffer;
+ for (unsigned Mask = 0x1;
+ Mask != SPIRV::CooperativeMatrixOperands::MatrixResultBFloat16ComponentsINTEL;
+ Mask <<= 1) {
+ if (MulAddOp & Mask) {
+ if (!Buffer.empty())
+ Buffer += '|';
+ Buffer += getSymbolicOperandMnemonic(
+ OperandCategory::CooperativeMatrixOperandsOperand, Mask);
+ }
+ }
+ OS << Buffer;
+ }
+ break;
+ }
default:
printRemainingVariableOps(MI, NumFixedOps, OS);
break;
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 45a49674d4ca21..28fcbda2d1df89 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1969,15 +1969,50 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call,
const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
unsigned Opcode =
SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
- bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR;
+ bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&
+ Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&
+ Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
unsigned ArgSz = Call->Arguments.size();
unsigned LiteralIdx = 0;
- if (Opcode == SPIRV::OpCooperativeMatrixLoadKHR && ArgSz > 3)
- LiteralIdx = 3;
- else if (Opcode == SPIRV::OpCooperativeMatrixStoreKHR && ArgSz > 4)
- LiteralIdx = 4;
+ switch(Opcode) {
+ // Memory operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixLoadKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreKHR:
+ LiteralIdx = ArgSz > 4 ? 4 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ LiteralIdx = ArgSz > 7 ? 7 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ LiteralIdx = ArgSz > 8 ? 8 : 0;
+ break;
+ // Cooperative Matrix Operands operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixMulAddKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
+ };
+
SmallVector<uint32_t, 1> ImmArgs;
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+ const uint32_t CacheLevel =
+ getConstFromIntrinsic(Call->Arguments[3], MRI);
+ auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)
+ .addUse(Call->Arguments[0]) // pointer
+ .addUse(Call->Arguments[1]) // rows
+ .addUse(Call->Arguments[2]) // columns
+ .addImm(CacheLevel) // cache level
+ .addUse(Call->Arguments[4]); // memory layout
+ if (ArgSz > 5)
+ MIB.addUse(Call->Arguments[5]); // stride
+ if (ArgSz > 6) {
+ const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI);
+ MIB.addImm(MemOp); // memory operand
+ }
+ return true;
+ }
if (LiteralIdx > 0)
ImmArgs.push_back(getConstFromIntrinsic(Call->Arguments[LiteralIdx], MRI));
Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index dc2da4a3a5647a..e29013d28aafe4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -695,6 +695,13 @@ defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreKHR", OpenCL_std, C
defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixMulAddKHR", OpenCL_std, CoopMatr, 3, 4, OpCooperativeMatrixMulAddKHR>;
defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLengthKHR", OpenCL_std, CoopMatr, 1, 1, OpCooperativeMatrixLengthKHR>;
+// Cooperative Matrix Intel builtin records:
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixPrefetchINTEL", OpenCL_std, CoopMatr, 5, 7, OpCooperativeMatrixPrefetchINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadCheckedINTEL", OpenCL_std, CoopMatr, 6, 8, OpCooperativeMatrixLoadCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixStoreCheckedINTEL", OpenCL_std, CoopMatr, 7, 9, OpCooperativeMatrixStoreCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixConstructCheckedINTEL", OpenCL_std, CoopMatr, 5, 5, OpCooperativeMatrixConstructCheckedINTEL>;
+defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixGetElementCoordINTEL", OpenCL_std, CoopMatr, 2, 2, OpCooperativeMatrixGetElementCoordINTEL>;
+
//===----------------------------------------------------------------------===//
// Class defining a work/sub group builtin that should be translated into a
// SPIR-V instruction using the defined properties.
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index e78fc5ce18707b..fb05c1fdbd1e3b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -51,6 +51,8 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
SPIRV::Extension::Extension::SPV_INTEL_subgroups},
{"SPV_INTEL_media_block_io",
SPIRV::Extension::Extension::SPV_INTEL_media_block_io},
+ {"SPV_INTEL_joint_matrix",
+ SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
{"SPV_KHR_uniform_group_instructions",
SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions},
{"SPV_KHR_no_integer_wrap_decoration",
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 53f1b644a94983..d95803fea56a58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -895,6 +895,23 @@ def OpCooperativeMatrixMulAddKHR: Op<4459, (outs ID:$res),
def OpCooperativeMatrixLengthKHR: Op<4460, (outs ID:$res), (ins TYPE:$type, ID:$coop_matr_type),
"$res = OpCooperativeMatrixLengthKHR $type $coop_matr_type">;
+// SPV_INTEL_joint_matrix
+def OpCooperativeMatrixLoadCheckedINTEL: Op<6193, (outs ID:$res),
+ (ins TYPE:$resType, ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+ "$res = OpCooperativeMatrixLoadCheckedINTEL $resType $pointer $xOffset $yOffset $memory_layout $height $width">;
+def OpCooperativeMatrixStoreCheckedINTEL: Op<6194, (outs),
+ (ins ID:$pointer, ID:$xOffset, ID:$yOffset, ID:$objectToStore, ID:$memory_layout, ID:$height, ID:$width, variable_ops),
+ "OpCooperativeMatrixStoreCheckedINTEL $pointer $xOffset $yOffset $objectToStore $memory_layout $height $width">;
+def OpCooperativeMatrixConstructCheckedINTEL: Op<6195, (outs ID:$res),
+ (ins TYPE:$resType, ID:$xOffset, ID:$yOffset, ID:$height, ID:$width, ID:$value),
+ "$res = OpCooperativeMatrixConstructCheckedINTEL $resType $xOffset $yOffset $height $width $value">;
+def OpCooperativeMatrixGetElementCoordINTEL: Op<6440, (outs ID:$res),
+ (ins TYPE:$resType, ID:$matrix, ID:$index),
+ "$res = OpCooperativeMatrixGetElementCoordINTEL $resType $matrix $index">;
+def OpCooperativeMatrixPrefetchINTEL: Op<6449, (outs),
+ (ins ID:$pointer, ID:$rows, ID:$columns, i32imm:$cacheLevel, ID:$memory_layout, variable_ops),
+ "OpCooperativeMatrixPrefetchINTEL $pointer $rows $columns $cacheLevel $memory_layout">;
+
// SPV_EXT_arithmetic_fence
def OpArithmeticFenceEXT: Op<6145, (outs ID:$res), (ins TYPE:$type, ID:$target),
"$res = OpArithmeticFenceEXT $type $target">;
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 20540814763157..ee7c08d324bb0c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1437,6 +1437,110 @@ void addInstrRequirements(const MachineInstr &MI,
Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
}
break;
+ case SPIRV::OpCooperativeMatrixMulAddKHR: {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+ report_fatal_error("Cooperative matrix instructions require the "
+ "following SPIR-V extension: "
+ "SPV_KHR_cooperative_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+ constexpr unsigned MulAddMaxSize = 6;
+ if (MI.getNumOperands() != MulAddMaxSize)
+ break;
+ const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
+ if (CoopOperands &
+ SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
+ }
+ if (CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixAAndBBFloat16ComponentsINTEL ||
+ CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixCBFloat16ComponentsINTEL ||
+ CoopOperands & SPIRV::CooperativeMatrixOperands::
+ MatrixResultBFloat16ComponentsINTEL) {
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
+ }
+ break;
+ }
+ case SPIRV::OpCooperativeMatrixLoadKHR:
+ case SPIRV::OpCooperativeMatrixStoreKHR:
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
+ report_fatal_error("Cooperative matrix instructions require the "
+ "following SPIR-V extension: "
+ "SPV_KHR_cooperative_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
+
+ // Check Layout operand in case if it's not a standart one and add the
+ // appropriate capability.
+ std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
+ {SPIRV::OpCooperativeMatrixLoadKHR, 3},
+ {SPIRV::OpCooperativeMatrixStoreKHR, 2},
+ {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
+ {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
+ {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}
+ };
+
+ const auto OpCode = MI.getOpcode();
+ const unsigned LayoutNum = LayoutToInstMap[OpCode];
+ Register RegLayout = MI.getOperand(LayoutNum).getReg();
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
+ if (MILayout->getOpcode() == SPIRV::OpConstantI) {
+ const unsigned LayoutVal = MILayout->getOperand(2).getImm();
+ if (LayoutVal == static_cast<unsigned>(
+ SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("PackedINTEL layout require the following SPIR-V "
+ "extension: SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
+ }
+ }
+
+ // Nothing to do.
+ if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||
+ OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
+ break;
+
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL "
+ "instructions require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
+ Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
+ break;
+ }
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+ break;
+ }
+ case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL"
+ " instructions require the following SPIR-V extension:"
+ " SPV_INTEL_joint_matrix", false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
+ break;
+ case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "
+ "following SPIR-V extension: SPV_INTEL_joint_matrix",
+ false);
+ Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
+ Reqs.addCapability(
+ SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
+ break;
case SPIRV::OpKill: {
Reqs.addCapability(SPIRV::Capability::Shader);
} break;
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index a3a88acdd6c6ae..745d1e1aec67aa 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -170,6 +170,8 @@ def GroupOperationOperand : OperandCategory;
def KernelEnqueueFlagsOperand : OperandCategory;
def KernelProfilingInfoOperand : OperandCategory;
def OpcodeOperand : OperandCategory;
+def CooperativeMatrixLayoutOperand : OperandCategory;
+def CooperativeMatrixOperandsOperand : OperandCategory;
//===----------------------------------------------------------------------===//
// Multiclass used to define Extesions enum values and at the same time
@@ -305,6 +307,7 @@ defm SPV_INTEL_global_variable_fpga_decorations : ExtensionOperand<110>;
defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>;
defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>;
defm SPV_EXT_optnone : ExtensionOperand<113>;
+defm SPV_INTEL_joint_matrix : ExtensionOperand<114>;
//===----------------------------------------------------------------------===//
// Multiclass used to define Capabilities enum values and at the same time
@@ -492,6 +495,12 @@ defm CacheControlsINTEL : CapabilityOperand<6441, 0, 0, [SPV_INTEL_cache_control
defm CooperativeMatrixKHR : CapabilityOperand<6022, 0, 0, [SPV_KHR_cooperative_matrix], []>;
defm ArithmeticFenceEXT : CapabilityOperand<6144, 0, 0, [SPV_EXT_arithmetic_fence], []>;
defm SplitBarrierINTEL : CapabilityOperand<6141, 0, 0, [SPV_INTEL_split_barrier], []>;
+defm CooperativeMatrixCheckedInstructionsINTEL : CapabilityOperand<6192, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixPrefetchINTEL : CapabilityOperand<6411, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm PackedCooperativeMatrixINTEL : CapabilityOperand<6434, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixInvocationInstructionsINTEL : CapabilityOperand<6435, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixTF32ComponentTypeINTEL : CapabilityOperand<6436, 0, 0, [SPV_INTEL_joint_matrix], []>;
+defm CooperativeMatrixBFloat16ComponentTypeINTEL : CapabilityOperand<6437, 0, 0, [SPV_INTEL_joint_matrix], []>;
//===----------------------------------------------------------------------===//
// Multiclass used to define SourceLanguage enum values and at the same time
@@ -1649,3 +1658,62 @@ defm GenericCastToPtr : OpcodeOperand<122>;
defm Bitcast : OpcodeOperand<124>;
defm ConvertPtrToU : OpcodeOperand<117>;
defm ConvertUToPtr : OpcodeOperand<120>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Layout enum values and at the
+// same time SymbolicOperand entries extensions and capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixLayout : GenericEnum, Operand<i32> {
+ let FilterClass = "CooperativeMatrixLayout";
+ let NameField = "Name";
+ let ValueField = "Value";
+}
+
+class CooperativeMatrixLayout<string name, bits<32> value> {
+ string Name = name;
+ bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixLayoutOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+ def : CooperativeMatrixLayout<NAME, value>;
+ defm : SymbolicOperandWithRequirements<CooperativeMatrixLayoutOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm RowMajorKHR : CooperativeMatrixLayoutOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm ColumnMajorKHR : CooperativeMatrixLayoutOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm PackedINTEL : CooperativeMatrixLayoutOperand<0x2, [SPV_INTEL_joint_matrix], [PackedCooperativeMatrixINTEL]>;
+
+//===----------------------------------------------------------------------===//
+// Multiclass used to define Cooperative Matrix Operands enum values and at the
+// same time SymbolicOperand entries with string mnemonics, extensions and
+// capabilities.
+//===----------------------------------------------------------------------===//
+
+def CooperativeMatrixOperands : GenericEnum, Operand<i32> {
+ let FilterClass = "CooperativeMatrixOperands";
+ let NameField = "Name";
+ let ValueField = "Value";
+ let PrintMethod = !strconcat("printSymbolicOperand<OperandCategory::", FilterClass, "Operand>");
+}
+
+class CooperativeMatrixOperands<string name, bits<32> value> {
+ string Name = name;
+ bits<32> Value = value;
+}
+
+multiclass CooperativeMatrixOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
+ def : CooperativeMatrixOperands<NAME, value>;
+ defm : SymbolicOperandWithRequirements<CooperativeMatrixOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+}
+
+defm NoneKHR : CooperativeMatrixOperandsOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixASignedComponentsKHR : CooperativeMatrixOperandsOperand<0x1, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixBSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x2, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixCSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x4, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixResultSignedComponentsKHR : CooperativeMatrixOperandsOperand<0x8, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm SaturatingAccumulationKHR : CooperativeMatrixOperandsOperand<0x10, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
+defm MatrixAAndBTF32ComponentsINTEL : CooperativeMatrixOperandsOperand<0x20, [SPV_INTEL_joint_matrix], [CooperativeMatrixTF32ComponentTypeINTEL]>;
+defm MatrixAAndBBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x40, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixCBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x80, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
+defm MatrixResultBFloat16ComponentsINTEL : CooperativeMatrixOperandsOperand<0x100, [SPV_INTEL_joint_matrix], [CooperativeMatrixBFloat16ComponentTypeINTEL]>;
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
new file mode 100644
index 00000000000000..192d769b8a3ce2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
@@ -0,0 +1,46 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: CooperativeMatrixBFloat16ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) #2
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
new file mode 100644
index 00000000000000..73ddaad0dbad19
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -0,0 +1,59 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const12:]] = OpConstant %[[#Int32Ty]] 12
+; CHECK-DAG: %[[#Const48:]] = OpConstant %[[#Int32Ty]] 48
+; CHECK-DAG: %[[#Const0:]] = OpConstant %[[#Int32Ty]] 0
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32Ty]] 3
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32Ty]] 1
+; CHECK-DAG: %[[#MatTy1:]] = OpTypeCooperativeMatrixKHR %[[#Int32Ty]] %[[#Const3]] %[[#Const12]] %[[#Const12]] %[[#Const2]]
+; CHECK-DAG: %[[#MatTy2:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const3]] %[[#Const12]] %[[#Const48]] %[[#Const0]]
+; CHECK-DAG: %[[#MatTy3:]] = OpTypeCooperativeMatrixKHR %[[#Int8Ty]] %[[#Const2]] %[[#Const48]] %[[#Const12]] %[[#Const1]]
+; CHECK: OpCooperativeMatrixConstructCheckedINTEL %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#MatTy3]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #2
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #2
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
new file mode 100644
index 00000000000000..4af85beb401ddc
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
@@ -0,0 +1,35 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability CooperativeMatrixInvocationInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#MatrixTy:]] = OpTypeCooperativeMatrixKHR
+
+; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]]
+; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]]
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+$_ZTSZZ23matrix_multiply_checked = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
+ %coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx)
+ ret void
+}
+; Function Attrs: convergent
+declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef)
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
new file mode 100644
index 00000000000000..50461af93084e1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
@@ -0,0 +1,78 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Capability PackedCooperativeMatrixINTEL
+; CHECK-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32Ty]] 2
+
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+$_ZTSZZ23matrix_multiply_checked = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) #1
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #1
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #1
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #1
+ ret void
+}
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #1
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #1
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #1
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #1
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
new file mode 100644
index 00000000000000..e68a5a2383e85f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
@@ -0,0 +1,34 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpExtension "SPV_INTEL_joint_matrix"
+
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]]
+; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) local_unnamed_addr #0 comdat {
+entry:
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) #1
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) #1
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) #1
+ ret void
+}
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
new file mode 100644
index 00000000000000..4fafca457110fa
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
@@ -0,0 +1,46 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
+
+; CHECK-DAG: Capability CooperativeMatrixKHR
+; CHECK-DAG: Extension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: Extension "SPV_INTEL_joint_matrix"
+; CHECK-DAG: OpCapability CooperativeMatrixTF32ComponentTypeINTEL
+
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL
+
+; ModuleID = 'test-matrix-opaque.bc'
+source_filename = "matrix-int8-test.cpp"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown"
+
+$_ZTSZZ15matrix_multiply = comdat any
+
+; Function Attrs: convergent norecurse
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+entry:
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) #2
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+
+
+attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
index 1c41c7331cda8a..e290c1eaeabad8 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix.ll
@@ -21,7 +21,7 @@
; CHECK: %[[#Load1:]] = OpCooperativeMatrixLoadKHR %[[#MatTy2]]
; CHECK: OpCooperativeMatrixLengthKHR %[[#Int32Ty]] %[[#MatTy2:]]
; CHECK: OpCooperativeMatrixLoadKHR %[[#MatTy3]]
-; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
+; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]] %[[#]] %[[#]] %[[#]] MatrixCSignedComponentsKHR|MatrixResultSignedComponentsKHR
; CHECK: OpCooperativeMatrixStoreKHR
define spir_kernel void @matr_mult(ptr addrspace(1) align 1 %_arg_accA, ptr addrspace(1) align 1 %_arg_accB, ptr addrspace(1) align 4 %_arg_accC, i64 %_arg_N, i64 %_arg_K) {
>From 5f459b8cd737fdad936db945848a03a590e025dc Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Wed, 4 Dec 2024 03:12:44 -0800
Subject: [PATCH 2/5] format
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
.../Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h | 4 +-
.../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 7 +--
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 51 +++++++++----------
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 38 +++++++-------
4 files changed, 52 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index 823c33ecb6bd38..2437fbb820a363 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -210,12 +210,12 @@ namespace Opcode {
namespace CooperativeMatrixLayout {
#define GET_CooperativeMatrixLayout_DECL
#include "SPIRVGenTables.inc"
-} // namespace Opcode
+} // namespace CooperativeMatrixLayout
namespace CooperativeMatrixOperands {
#define GET_CooperativeMatrixOperands_DECL
#include "SPIRVGenTables.inc"
-} // namespace Opcode
+} // namespace CooperativeMatrixOperands
struct ExtendedBuiltin {
StringRef Name;
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index d05a0e87ca870e..2ee0c79b8f7c12 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -220,12 +220,13 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
const unsigned MulAddOp = MI->getOperand(FirstVariableIndex).getImm();
if (MulAddOp == 0) {
printSymbolicOperand<
- OperandCategory::CooperativeMatrixOperandsOperand>(
- MI, FirstVariableIndex, OS);
+ OperandCategory::CooperativeMatrixOperandsOperand>(
+ MI, FirstVariableIndex, OS);
} else {
std::string Buffer;
for (unsigned Mask = 0x1;
- Mask != SPIRV::CooperativeMatrixOperands::MatrixResultBFloat16ComponentsINTEL;
+ Mask != SPIRV::CooperativeMatrixOperands::
+ MatrixResultBFloat16ComponentsINTEL;
Mask <<= 1) {
if (MulAddOp & Mask) {
if (!Buffer.empty())
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 28fcbda2d1df89..9b6c2a849edce2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1974,42 +1974,41 @@ static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call,
Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
unsigned ArgSz = Call->Arguments.size();
unsigned LiteralIdx = 0;
- switch(Opcode) {
- // Memory operand is optional and is literal.
- case SPIRV::OpCooperativeMatrixLoadKHR:
- LiteralIdx = ArgSz > 3 ? 3 : 0;
- break;
- case SPIRV::OpCooperativeMatrixStoreKHR:
- LiteralIdx = ArgSz > 4 ? 4 : 0;
- break;
- case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
- LiteralIdx = ArgSz > 7 ? 7 : 0;
- break;
- case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
- LiteralIdx = ArgSz > 8 ? 8 : 0;
- break;
- // Cooperative Matrix Operands operand is optional and is literal.
- case SPIRV::OpCooperativeMatrixMulAddKHR:
- LiteralIdx = ArgSz > 3 ? 3 : 0;
- break;
+ switch (Opcode) {
+ // Memory operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixLoadKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreKHR:
+ LiteralIdx = ArgSz > 4 ? 4 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ LiteralIdx = ArgSz > 7 ? 7 : 0;
+ break;
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ LiteralIdx = ArgSz > 8 ? 8 : 0;
+ break;
+ // Cooperative Matrix Operands operand is optional and is literal.
+ case SPIRV::OpCooperativeMatrixMulAddKHR:
+ LiteralIdx = ArgSz > 3 ? 3 : 0;
+ break;
};
SmallVector<uint32_t, 1> ImmArgs;
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
- const uint32_t CacheLevel =
- getConstFromIntrinsic(Call->Arguments[3], MRI);
+ const uint32_t CacheLevel = getConstFromIntrinsic(Call->Arguments[3], MRI);
auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)
- .addUse(Call->Arguments[0]) // pointer
- .addUse(Call->Arguments[1]) // rows
- .addUse(Call->Arguments[2]) // columns
- .addImm(CacheLevel) // cache level
- .addUse(Call->Arguments[4]); // memory layout
+ .addUse(Call->Arguments[0]) // pointer
+ .addUse(Call->Arguments[1]) // rows
+ .addUse(Call->Arguments[2]) // columns
+ .addImm(CacheLevel) // cache level
+ .addUse(Call->Arguments[4]); // memory layout
if (ArgSz > 5)
MIB.addUse(Call->Arguments[5]); // stride
if (ArgSz > 6) {
const uint32_t MemOp = getConstFromIntrinsic(Call->Arguments[6], MRI);
- MIB.addImm(MemOp); // memory operand
+ MIB.addImm(MemOp); // memory operand
}
return true;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index ee7c08d324bb0c..616e9d348e15a1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1441,7 +1441,8 @@ void addInstrRequirements(const MachineInstr &MI,
if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
report_fatal_error("Cooperative matrix instructions require the "
"following SPIR-V extension: "
- "SPV_KHR_cooperative_matrix", false);
+ "SPV_KHR_cooperative_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
constexpr unsigned MulAddMaxSize = 6;
@@ -1455,11 +1456,11 @@ void addInstrRequirements(const MachineInstr &MI,
SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
}
if (CoopOperands & SPIRV::CooperativeMatrixOperands::
- MatrixAAndBBFloat16ComponentsINTEL ||
- CoopOperands & SPIRV::CooperativeMatrixOperands::
- MatrixCBFloat16ComponentsINTEL ||
+ MatrixAAndBBFloat16ComponentsINTEL ||
+ CoopOperands &
+ SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
CoopOperands & SPIRV::CooperativeMatrixOperands::
- MatrixResultBFloat16ComponentsINTEL) {
+ MatrixResultBFloat16ComponentsINTEL) {
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
@@ -1474,19 +1475,19 @@ void addInstrRequirements(const MachineInstr &MI,
if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
report_fatal_error("Cooperative matrix instructions require the "
"following SPIR-V extension: "
- "SPV_KHR_cooperative_matrix", false);
+ "SPV_KHR_cooperative_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
// Check Layout operand in case if it's not a standart one and add the
// appropriate capability.
std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
- {SPIRV::OpCooperativeMatrixLoadKHR, 3},
- {SPIRV::OpCooperativeMatrixStoreKHR, 2},
- {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
- {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
- {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}
- };
+ {SPIRV::OpCooperativeMatrixLoadKHR, 3},
+ {SPIRV::OpCooperativeMatrixStoreKHR, 2},
+ {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
+ {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
+ {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}};
const auto OpCode = MI.getOpcode();
const unsigned LayoutNum = LayoutToInstMap[OpCode];
@@ -1495,11 +1496,12 @@ void addInstrRequirements(const MachineInstr &MI,
MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
if (MILayout->getOpcode() == SPIRV::OpConstantI) {
const unsigned LayoutVal = MILayout->getOperand(2).getImm();
- if (LayoutVal == static_cast<unsigned>(
- SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
+ if (LayoutVal ==
+ static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
report_fatal_error("PackedINTEL layout require the following SPIR-V "
- "extension: SPV_INTEL_joint_matrix", false);
+ "extension: SPV_INTEL_joint_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
}
@@ -1513,7 +1515,8 @@ void addInstrRequirements(const MachineInstr &MI,
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL "
"instructions require the following SPIR-V extension: "
- "SPV_INTEL_joint_matrix", false);
+ "SPV_INTEL_joint_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
@@ -1527,7 +1530,8 @@ void addInstrRequirements(const MachineInstr &MI,
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL"
" instructions require the following SPIR-V extension:"
- " SPV_INTEL_joint_matrix", false);
+ " SPV_INTEL_joint_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
>From cc73812a71bcefae5f3e0ae230d634fae71f2564 Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Wed, 4 Dec 2024 06:20:45 -0800
Subject: [PATCH 3/5] apply comments, improve validation, add negative tests
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 37 +++++++---
.../cooperative_matrix_bf16.ll | 42 ++++-------
.../cooperative_matrix_checked.ll | 43 ++++--------
.../cooperative_matrix_get_coord.ll | 23 ++----
.../cooperative_matrix_packed.ll | 70 +++++++------------
.../cooperative_matrix_prefetch.ll | 29 +++-----
.../cooperative_matrix_tf32.ll | 42 ++++-------
7 files changed, 113 insertions(+), 173 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 616e9d348e15a1..1a125c44750e6a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1451,6 +1451,9 @@ void addInstrRequirements(const MachineInstr &MI,
const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
if (CoopOperands &
SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
+ report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
+ "require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix", false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
@@ -1461,6 +1464,9 @@ void addInstrRequirements(const MachineInstr &MI,
SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
CoopOperands & SPIRV::CooperativeMatrixOperands::
MatrixResultBFloat16ComponentsINTEL) {
+ report_fatal_error("***BF16ComponentsINTEL type interpretations require "
+ "the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix", false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
@@ -1480,7 +1486,7 @@ void addInstrRequirements(const MachineInstr &MI,
Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
- // Check Layout operand in case if it's not a standart one and add the
+ // Check Layout operand in case if it's not a standard one and add the
// appropriate capability.
std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
{SPIRV::OpCooperativeMatrixLoadKHR, 3},
@@ -1512,11 +1518,24 @@ void addInstrRequirements(const MachineInstr &MI,
OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
break;
- if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
- report_fatal_error("OpCooperativeMatrix[Load/Store]CheckedINTEL "
- "instructions require the following SPIR-V extension: "
- "SPV_INTEL_joint_matrix",
- false);
+ std::string InstName;
+ switch(OpCode) {
+ case SPIRV::OpCooperativeMatrixPrefetchINTEL:
+ InstName = "OpCooperativeMatrixPrefetchINTEL";
+ break;
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ InstName = "OpCooperativeMatrixLoadCheckedINTEL";
+ break;
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ InstName = "OpCooperativeMatrixStoreCheckedINTEL";
+ break;
+ }
+
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {
+ const std::string ErrorMsg = InstName + " instruction requires the "
+ "following SPIR-V extension: SPV_INTEL_joint_matrix";
+ report_fatal_error(ErrorMsg.c_str(), false);
+ }
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
@@ -1528,9 +1547,9 @@ void addInstrRequirements(const MachineInstr &MI,
}
case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
- report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL"
- " instructions require the following SPIR-V extension:"
- " SPV_INTEL_joint_matrix",
+ report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL "
+ "instructions require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix",
false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
index 192d769b8a3ce2..6d448040059ec4 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: ***BF16ComponentsINTEL type interpretations require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixKHR
@@ -7,40 +11,22 @@
; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBBFloat16ComponentsINTEL
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
entry:
%matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
- %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
- %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
- %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64) #2
- tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1)
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K)
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 64)
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1)
ret void
}
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef)
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i16, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i16, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32) #1
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", float, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #2 = { convergent }
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2), i32, i64, i32)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
index 73ddaad0dbad19..9d89707fd3178a 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixKHR
@@ -21,39 +25,22 @@
; CHECK: OpCooperativeMatrixMulAddKHR %[[#MatTy1]]
; CHECK: OpCooperativeMatrixStoreCheckedINTEL
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
entry:
- %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #2
- %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #2
- %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #2
- %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #2
- tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #2
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1)
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 1, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K)
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 0, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1)
ret void
}
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef)
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #2 = { convergent }
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
index 4af85beb401ddc..c9cbe239c7ceeb 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixGetElementCoordINTEL requires the following SPIR-V extension: SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixKHR
@@ -9,27 +13,12 @@
; CHECK: %[[#Matrix:]] = OpCompositeConstruct %[[#MatrixTy]]
; CHECK: %[[#]] = OpCooperativeMatrixGetElementCoordINTEL %[[#]] %[[#Matrix]] %[[#]]
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-$_ZTSZZ23matrix_multiply_checked = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(i32 noundef %_idx, i32 noundef %_arg_Initvalue) {
entry:
%matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float 0.0)
%coord = tail call spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %matrixC, i32 noundef %_idx)
ret void
}
-; Function Attrs: convergent
declare dso_local spir_func <2 x i32> @_Z45__spirv_CooperativeMatrixGetElementCoordINTEL(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #1
-
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent }
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
index 50461af93084e1..806641a29520b4 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: PackedINTEL layout require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixKHR
@@ -15,64 +19,40 @@
; CHECK: OpCooperativeMatrixLoadCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
; CHECK: OpCooperativeMatrixStoreCheckedINTEL %[[#]] %[[#]] %[[#]] %[[#]] %[[#Const2]]
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-$_ZTSZZ23matrix_multiply_checked = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
entry:
- %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue) #1
- %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #1
- %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #1
- %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
- tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #1
+ %matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 %_arg_Initvalue)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1)
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K)
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1)
ret void
}
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ23matrix_multiply_checked(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, i32 noundef %_arg_Initvalue) {
entry:
- %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue) #1
- %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1) #1
- %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K) #1
- %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12) #1
- tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1) #1
+ %matrixC = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef 4, i32 noundef 4, i32 noundef 12, i32 noundef 12, i32 noundef %_arg_Initvalue)
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 12, i32 noundef 48, i64 noundef %_arg_K, i32 noundef 1)
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(1) noundef %_arg_accB, i32 noundef 0, i32 noundef 0, i32 noundef 2, i32 noundef 48, i32 noundef 12, i64 noundef %_arg_K)
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %matrixC, i32 noundef 12)
+ tail call spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(1) noundef %_arg_accC, i32 noundef 0, i32 noundef 0, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %res, i32 noundef 2, i32 noundef 12, i32 noundef 12, i64 noundef %_arg_N, i32 noundef 1)
ret void
}
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) #1
-
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32)
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z46__spirv_CooperativeMatrixConstructCheckedINTEL(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_1(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z41__spirv_CooperativeMatrixLoadCheckedINTEL_2(ptr addrspace(4) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", i8, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+declare dso_local spir_func void @_Z42__spirv_CooperativeMatrixStoreCheckedINTEL(ptr addrspace(4) noundef, i32 noundef, i32 noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent }
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
index e68a5a2383e85f..90c71ebfb660e4 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixPrefetchINTEL instruction requires the following SPIR-V extension: SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixPrefetchINTEL
@@ -8,27 +12,16 @@
; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]]
; CHECK-DAG: OpCooperativeMatrixPrefetchINTEL %[[#]] %[[#]] %[[#]] 0 %[[#]] %[[#]] 1
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_K) {
entry:
- tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0) #1
- tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K) #1
- tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1) #1
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef %_arg_accA, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0)
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef %_arg_accB, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K)
+ tail call spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef %_arg_accC, i32 noundef 12, i32 noundef 48, i32 noundef 0, i32 noundef 0, i64 noundef %_arg_K, i32 1)
ret void
}
-declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
-
-declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef) local_unnamed_addr #1
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiii(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
-declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #1
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiil(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef)
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent }
+declare dso_local spir_func void @_Z38__spirv_CooperativeMatrixPrefetchINTELPU3AS4ciiiili(ptr addrspace(1) noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i64 noundef, i32 noundef)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
index 4fafca457110fa..15a57621ccde0d 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
@@ -1,3 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; CHECK-ERROR: LLVM ERROR: MatrixAAndBTF32ComponentsINTEL type interpretation require the following SPIR-V extension: SPV_INTEL_joint_matrix
+
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
; CHECK-DAG: Capability CooperativeMatrixKHR
@@ -7,40 +11,22 @@
; CHECK: OpCooperativeMatrixMulAddKHR %[[#]] %[[#]] %[[#]] %[[#]] MatrixAAndBTF32ComponentsINTEL
-; ModuleID = 'test-matrix-opaque.bc'
-source_filename = "matrix-int8-test.cpp"
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-
-$_ZTSZZ15matrix_multiply = comdat any
-
-; Function Attrs: convergent norecurse
-define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) local_unnamed_addr #0 comdat {
+define weak_odr dso_local spir_kernel void @_ZTSZZ15matrix_multiply(ptr addrspace(1) noundef align 1 %_arg_accA, ptr addrspace(1) noundef align 1 %_arg_accB, ptr addrspace(1) noundef align 1 %_arg_accC, i64 noundef %_arg_N, i64 noundef %_arg_K, float noundef %_arg_Initvalue) {
entry:
%matrixC = tail call spir_func target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float %_arg_Initvalue)
- %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 2, i64 noundef %_arg_K, i32 noundef 1) #2
- %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 2, i64 noundef %_arg_K) #2
- %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32) #2
- tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 2, i64 noundef %_arg_N, i32 noundef 1) #2
+ %matrixA = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accA, i32 noundef 0, i64 noundef %_arg_K, i32 noundef 1)
+ %matrixB = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(1) noundef %_arg_accB, i32 noundef 1, i64 noundef %_arg_K)
+ %res = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef %matrixA, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef %matrixB, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %matrixC, i32 noundef 32)
+ tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef %res, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1)
ret void
}
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef) local_unnamed_addr #1
-
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32) #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(float noundef)
-; Function Attrs: convergent
-declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64) #1
+declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHR(target("spirv.CooperativeMatrixKHR", float, 3, 12, 48, 0) noundef, target("spirv.CooperativeMatrixKHR", float, 2, 48, 12, 1) noundef, target("spirv.CooperativeMatrixKHR", float, 3, 12, 12, 2) noundef, i32 noundef)
-; Function Attrs: convergent
-declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32) #1
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 48, 0) @_Z32__spirv_CooperativeMatrixLoadKHR_1(ptr addrspace(3), i32, i64, i32)
+declare dso_local spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 48, 12, 1) @_Z32__spirv_CooperativeMatrixLoadKHR_2(ptr addrspace(3), i32, i64)
-attributes #0 = { convergent norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="matrix-int8-test.cpp" "uniform-work-group-size"="true" }
-attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #2 = { convergent }
+declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(3), target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), i32, i64, i32)
>From b88548217c6fe22266b1c12a11953c038c59ef94 Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Wed, 4 Dec 2024 06:23:07 -0800
Subject: [PATCH 4/5] fix tests
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
.../SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll | 2 +-
.../SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll | 2 +-
.../SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll | 2 +-
.../SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll | 2 +-
.../SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll | 2 +-
.../SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
index 6d448040059ec4..c0b23a324992e0 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_bf16.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: ***BF16ComponentsINTEL type interpretations require the following SPIR-V extension: SPV_INTEL_joint_matrix
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
index 9d89707fd3178a..f5e1907792fcea 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
index c9cbe239c7ceeb..e7ec5e186d16a6 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_get_coord.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixGetElementCoordINTEL requires the following SPIR-V extension: SPV_INTEL_joint_matrix
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
index 806641a29520b4..8b336df1c2c208 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_packed.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: PackedINTEL layout require the following SPIR-V extension: SPV_INTEL_joint_matrix
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
index 90c71ebfb660e4..8573e092844031 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixPrefetchINTEL instruction requires the following SPIR-V extension: SPV_INTEL_joint_matrix
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
index 15a57621ccde0d..d3ada306d2c2f8 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_tf32.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
; CHECK-ERROR: LLVM ERROR: MatrixAAndBTF32ComponentsINTEL type interpretation require the following SPIR-V extension: SPV_INTEL_joint_matrix
>From 2c8b86ab6b488483e739e86944a2435377293e12 Mon Sep 17 00:00:00 2001
From: "Sidorov, Dmitry" <dmitry.sidorov at intel.com>
Date: Wed, 4 Dec 2024 06:49:05 -0800
Subject: [PATCH 5/5] format and fix
Signed-off-by: Sidorov, Dmitry <dmitry.sidorov at intel.com>
---
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 41 +++++++++++--------
.../cooperative_matrix_checked.ll | 2 +-
2 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 1a125c44750e6a..a9829ead0cf1fa 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -1451,9 +1451,11 @@ void addInstrRequirements(const MachineInstr &MI,
const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
if (CoopOperands &
SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
- report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
- "require the following SPIR-V extension: "
- "SPV_INTEL_joint_matrix", false);
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
+ "require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
@@ -1464,9 +1466,11 @@ void addInstrRequirements(const MachineInstr &MI,
SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
CoopOperands & SPIRV::CooperativeMatrixOperands::
MatrixResultBFloat16ComponentsINTEL) {
- report_fatal_error("***BF16ComponentsINTEL type interpretations require "
- "the following SPIR-V extension: "
- "SPV_INTEL_joint_matrix", false);
+ if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
+ report_fatal_error("***BF16ComponentsINTEL type interpretations "
+ "require the following SPIR-V extension: "
+ "SPV_INTEL_joint_matrix",
+ false);
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
Reqs.addCapability(
SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
@@ -1519,21 +1523,22 @@ void addInstrRequirements(const MachineInstr &MI,
break;
std::string InstName;
- switch(OpCode) {
- case SPIRV::OpCooperativeMatrixPrefetchINTEL:
- InstName = "OpCooperativeMatrixPrefetchINTEL";
- break;
- case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
- InstName = "OpCooperativeMatrixLoadCheckedINTEL";
- break;
- case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
- InstName = "OpCooperativeMatrixStoreCheckedINTEL";
- break;
+ switch (OpCode) {
+ case SPIRV::OpCooperativeMatrixPrefetchINTEL:
+ InstName = "OpCooperativeMatrixPrefetchINTEL";
+ break;
+ case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
+ InstName = "OpCooperativeMatrixLoadCheckedINTEL";
+ break;
+ case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
+ InstName = "OpCooperativeMatrixStoreCheckedINTEL";
+ break;
}
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {
- const std::string ErrorMsg = InstName + " instruction requires the "
- "following SPIR-V extension: SPV_INTEL_joint_matrix";
+ const std::string ErrorMsg =
+ InstName + " instruction requires the "
+ "following SPIR-V extension: SPV_INTEL_joint_matrix";
report_fatal_error(ErrorMsg.c_str(), false);
}
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
index f5e1907792fcea..a4b2c4be5084b0 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll
@@ -1,6 +1,6 @@
; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
-; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension:SPV_INTEL_joint_matrix
+; CHECK-ERROR: LLVM ERROR: OpCooperativeMatrixConstructCheckedINTEL instructions require the following SPIR-V extension: SPV_INTEL_joint_matrix
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix %s -o - | FileCheck %s
More information about the llvm-commits
mailing list