[llvm] [SPIR-V] Access-chain aggregate pointers to element 0 for cooperative matrix load/store (PR #202050)
Julian Klappenbach via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 21 18:26:14 PDT 2026
https://github.com/jklappenbach updated https://github.com/llvm/llvm-project/pull/202050
>From 6bdc937544fe316d307ca57500ba76dd24f162f4 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:12 -0400
Subject: [PATCH 1/7] [SPIRV] Cooperative matrix under the Vulkan/Shader flavor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Make SPV_KHR_cooperative_matrix reachable from the Vulkan/Shader flavor (the
OpenCL __spirv_* builtin path is isShader()-gated off), mirroring the texture /
ray-query intrinsic + GlobalISel pattern:
- IntrinsicsSPIRV.td: llvm.spv.cooperative.matrix.{load,store,muladd,splat};
the opaque matrix is carried as llvm_any_ty (overloaded per concrete shape).
- SPIRVInstructionSelector.cpp: GlobalISel selection for the four ops
(load/muladd/splat via selectOpWithSrcs, store via selectCoopMatrixStore).
memory-layout/stride are <id> constant operands; float MulAdd omits the
integer-only signedness literal; splat = single-scalar OpCompositeConstruct.
- SPIRVModuleAnalysis.{cpp,h}: cooperative matrix under Shader mandates the
Vulkan memory model (spirv-val rejects Shader + CooperativeMatrixKHR under
GLSL450). Rather than guess up front, derive the model from the actual
requirement: after collectReqs, if a Shader module requires CooperativeMatrixKHR
(RequirementHandler::isCapabilityRequired, added here), upgrade GLSL450 ->
VulkanKHR (+ VulkanMemoryModelKHR capability + SPV_KHR_vulkan_memory_model
extension, OpMemoryModel Logical VulkanKHR), unless !spirv.MemoryModel set one
explicitly. Non-coop shaders keep GLSL450.
The OpTypeCooperativeMatrixKHR opaque type needs no backend change — the
existing BuiltinType machinery lowers target("spirv.CooperativeMatrixKHR", elem,
scope, rows, cols, use) flavor-agnostically.
---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 13 +++-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 30 +++++++
.../cooperative_matrix_kernel.ll | 78 +++++++++++++++++++
.../cooperative_matrix_ops_vulkan.ll | 53 +++++++++++++
.../cooperative_matrix_type_vulkan.ll | 22 ++++++
5 files changed, 195 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 6e4cf8f7e72dc..7f8e23da39b61 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -365,5 +365,16 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
def int_spv_unpackhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_i32_ty], [IntrNoMem]>;
def int_spv_packhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
-
+ // SPV_KHR_cooperative_matrix operations under the Vulkan/Shader flavor,
+ // reached through llvm.spv intrinsics and GlobalISel selection because the
+ // __spirv_* builtin path is OpenCL-only. The matrix value is the opaque
+ // target("spirv.CooperativeMatrixKHR", ...) type, carried as llvm_any_ty.
+ def int_spv_cooperative_matrix_load
+ : Intrinsic<[llvm_any_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty]>;
+ def int_spv_cooperative_matrix_store
+ : Intrinsic<[], [llvm_anyptr_ty, llvm_any_ty, llvm_i32_ty, llvm_i32_ty]>;
+ def int_spv_cooperative_matrix_muladd
+ : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_any_ty]>;
+ def int_spv_cooperative_matrix_splat
+ : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index cd99015a61ba9..12bd09ede4ade 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -452,6 +452,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectImageWriteIntrinsic(MachineInstr &I) const;
+ bool selectCoopMatrixStore(MachineInstr &I) const;
bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1609,6 +1610,17 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
+bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
+ // Operands 1..: pointer, matrix, memory_layout, stride.
+ // OpCooperativeMatrixStoreKHR has no result.
+ auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+ TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
+ for (unsigned i = 1; i < I.getNumOperands(); ++i)
+ MIB.addUse(I.getOperand(i).getReg());
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I,
@@ -4947,6 +4959,24 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
report_fatal_error("incompatible result and operand types in a bitcast");
return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
}
+ case Intrinsic::spv_cooperative_matrix_load:
+ // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
+ return selectOpWithSrcs(ResVReg, ResType, I,
+ {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ I.getOperand(4).getReg()},
+ SPIRV::OpCooperativeMatrixLoadKHR);
+ case Intrinsic::spv_cooperative_matrix_store:
+ return selectCoopMatrixStore(I);
+ case Intrinsic::spv_cooperative_matrix_muladd:
+ // OpCooperativeMatrixMulAddKHR A B C; no operands literal for float matmul.
+ return selectOpWithSrcs(ResVReg, ResType, I,
+ {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ I.getOperand(4).getReg()},
+ SPIRV::OpCooperativeMatrixMulAddKHR);
+ case Intrinsic::spv_cooperative_matrix_splat:
+ // OpCompositeConstruct from a scalar broadcasts the accumulator.
+ return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
+ SPIRV::OpCompositeConstruct);
case Intrinsic::spv_unref_global:
case Intrinsic::spv_init_global: {
MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
new file mode 100644
index 0000000000000..d968647eecc8c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
@@ -0,0 +1,78 @@
+; A GLCompute kernel that runs a cooperative-matrix matmul tile against
+; descriptor-bound storage buffers under the Vulkan flavor. A and B are read-only
+; StorageBuffers and C a writable StorageBuffer, each a VulkanBuffer handle
+; decorated DescriptorSet and Binding and accessed through
+; llvm.spv.resource.handlefrombinding and getpointer. The kernel loads A (use 0)
+; and B (use 1) as cooperative matrices, mul-adds into a zero accumulator C
+; (use 2), and stores C. The module must pass spirv-val --target-env vulkan1.3.
+
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpEntryPoint GLCompute %[[#entry:]] "main"
+; CHECK-DAG: OpDecorate %[[#A:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#A]] Binding 0
+; CHECK-DAG: OpDecorate %[[#B:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#B]] Binding 1
+; CHECK-DAG: OpDecorate %[[#C:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#C]] Binding 2
+; CHECK: %[[#MA:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MB:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MC0:]] = OpCompositeConstruct
+; CHECK: %[[#MC:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#MA]] %[[#MB]] %[[#MC0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#MC]]
+
+ at .str.a = private unnamed_addr constant [2 x i8] c"a\00", align 1
+ at .str.b = private unnamed_addr constant [2 x i8] c"b\00", align 1
+ at .str.c = private unnamed_addr constant [2 x i8] c"c\00", align 1
+
+define void @main() local_unnamed_addr #0 {
+entry:
+ ; A: read-only StorageBuffer, binding 0 -> MatrixA (use 0)
+ %ha = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+ i32 0, i32 0, i32 1, i32 0, ptr nonnull @.str.a)
+ %pa = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 0) %ha, i32 0)
+ %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pa, i32 0, i32 16)
+
+ ; B: read-only StorageBuffer, binding 1 -> MatrixB (use 1)
+ %hb = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+ i32 0, i32 1, i32 1, i32 0, ptr nonnull @.str.b)
+ %pb = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 0) %hb, i32 0)
+ %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pb, i32 0, i32 16)
+
+ ; C = A*B + 0 (accumulator, use 2), stored to writable StorageBuffer binding 2
+ %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+ %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.muladd(
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+ %hc = tail call target("spirv.VulkanBuffer", [0 x float], 12, 1)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_1t(
+ i32 0, i32 2, i32 1, i32 0, ptr nonnull @.str.c)
+ %pc = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_1t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 1) %hc, i32 0)
+ call void @llvm.spv.cooperative.matrix.store(
+ ptr addrspace(11) %pc,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+ i32 0, i32 16)
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
new file mode 100644
index 0000000000000..365fa2cd9a404
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
@@ -0,0 +1,53 @@
+; The cooperative-matrix operations (load, store, mul-add, splat-construct) under
+; the Vulkan/Shader flavor, reached via llvm.spv.cooperative.matrix.* intrinsics
+; and GlobalISel selection because the OpenCL __spirv_CooperativeMatrix* builtin
+; path is OpenCL-only.
+;
+; A single matmul tile fragment: C = A * B + 0. A is MatrixA (use 0), B is MatrixB
+; (use 1), C the accumulator (use 2); all 16x16 f32, Subgroup scope (3). Memory
+; layout 0 = RowMajorKHR, stride 16.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; The module sets the Vulkan memory model explicitly via spirv.MemoryModel
+; metadata (spirv-val rejects Shader + CooperativeMatrixKHR under GLSL450).
+; CHECK-DAG: OpCapability VulkanMemoryModelKHR
+; CHECK-DAG: OpExtension "SPV_KHR_vulkan_memory_model"
+; CHECK-DAG: OpMemoryModel Logical VulkanKHR
+; CHECK-DAG: %[[#F32:]] = OpTypeFloat 32
+; The three matrix types (A use 0, B use 1, C accumulator use 2) all lower to
+; OpTypeCooperativeMatrixKHR over the f32 component type.
+; CHECK-DAG: OpTypeCooperativeMatrixKHR %[[#F32]]
+; Data flow: mul-add consumes both loaded operands plus the splat-constructed
+; accumulator; the store consumes the mul-add result.
+; CHECK: %[[#A:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#B:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#C0:]] = OpCompositeConstruct
+; CHECK: %[[#C:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#A]] %[[#B]] %[[#C0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#C]]
+
+define spir_func void @matmul_tile(ptr %pa, ptr %pb, ptr %pc) {
+entry:
+ %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr %pa, i32 0, i32 16)
+ %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+ @llvm.spv.cooperative.matrix.load(ptr %pb, i32 0, i32 16)
+ %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+ %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.muladd(
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+ call void @llvm.spv.cooperative.matrix.store(
+ ptr %pc,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+ i32 0, i32 16)
+ ret void
+}
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
new file mode 100644
index 0000000000000..e6d0aac36812f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
@@ -0,0 +1,22 @@
+; Check that the opaque CooperativeMatrix type under the Vulkan/Shader flavor,
+; target("spirv.CooperativeMatrixKHR", elem, scope, rows, cols, use), lowers to
+; OpTypeCooperativeMatrixKHR, gated behind the CooperativeMatrixKHR capability and
+; the SPV_KHR_cooperative_matrix extension, and that it errors cleanly without it.
+; This is a text-emission check only; the type is forced via a by-value parameter,
+; so it is not run through spirv-val here.
+
+; RUN: not llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-ERROR: LLVM ERROR: OpTypeCooperativeMatrixKHR type requires the following SPIR-V extension: SPV_KHR_cooperative_matrix
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: {{%[0-9]+}} = OpTypeCooperativeMatrixKHR
+
+; A by-value parameter of the opaque type forces OpTypeCooperativeMatrixKHR into the
+; module (referenced by OpTypeFunction, so it is not eliminated like an unused local).
+define spir_func void @use_coopmat(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %m) {
+entry:
+ ret void
+}
>From 76913939000c3548bbb9280c680e72e8738462ee Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:52 -0400
Subject: [PATCH 2/7] [SPIR-V] Access-chain aggregate pointers to element 0 for
cooperative matrix load/store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
OpCooperativeMatrixLoadKHR / OpCooperativeMatrixStoreKHR require the Pointer to
point to a scalar or vector — the tile's element type. When the source is a
workgroup-shared array tile, however, the pointer reaching the selector is a
pointer to the whole `[N x T]` array: in opaque-pointer IR `&arr[0]` is the same
SSA value as `&arr`, and a zero-index element GEP is simplified back to the array
base during SPIRVEmitIntrinsics. The cooperative-matrix op was then emitted with
an array pointer, which spirv-val rejects ("Pointer's Type must be a scalar or
vector type").
Index such a pointer to its first element with an OpAccessChain before the
cooperative-matrix op, so it receives the required scalar pointer. Index 0 selects
the same address the MemoryLayout/Stride operands then walk from, so semantics are
unchanged. Pointers that are already element-typed (e.g. a StorageBuffer access
chain, or a dynamic-offset Workgroup access chain) are not aggregates and pass
through untouched.
Verified with llc + spirv-val on a Workgroup-tile cooperative-matrix load, and a
full LDS-staged GEMM (CoopStage copy -> Barrier -> load(Shared) -> mma) computing
bit-exact results on RADV / gfx1151.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 45 ++++++++++++++++++-
.../cooperative_matrix_workgroup_source.ll | 38 ++++++++++++++++
2 files changed, 81 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 12bd09ede4ade..5d3fdc8988f61 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -453,6 +453,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
MachineInstr &I) const;
bool selectImageWriteIntrinsic(MachineInstr &I) const;
bool selectCoopMatrixStore(MachineInstr &I) const;
+ Register coopMatrixElementPtr(Register PtrReg, MachineInstr &I) const;
bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1610,12 +1611,51 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
+// OpCooperativeMatrixLoad/StoreKHR require the pointer to point to the element
+// type. A workgroup array tile reaches the selector as a pointer to the whole
+// array, so index it to its first element to get the scalar pointer the op
+// requires. Returns PtrReg unchanged when it is not an aggregate.
+Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
+ MachineInstr &I) const {
+ SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
+ if (!PtrType)
+ return PtrReg;
+ SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
+ if (!PointeeType || PointeeType->getOpcode() != SPIRV::OpTypeArray)
+ return PtrReg;
+ SPIRVTypeInst ElemType =
+ GR.getSPIRVTypeForVReg(PointeeType->getOperand(1).getReg());
+ if (!ElemType)
+ return PtrReg;
+ SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(PtrReg);
+ MachineIRBuilder MIRBuilder(I);
+ SPIRVTypeInst ElemPtrType =
+ GR.getOrCreateSPIRVPointerType(ElemType, MIRBuilder, SC);
+ SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
+ Register Zero = buildZerosVal(I32Type, I);
+ Register NewPtr = MRI->createVirtualRegister(GR.getRegClass(ElemPtrType));
+ GR.assignSPIRVTypeToVReg(ElemPtrType, NewPtr, *I.getParent()->getParent());
+ unsigned Opcode =
+ STI.isLogicalSPIRV() ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
+ BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+ .addDef(NewPtr)
+ .addUse(GR.getSPIRVTypeID(ElemPtrType))
+ .addUse(PtrReg)
+ .addUse(Zero)
+ .constrainAllUses(TII, TRI, RBI);
+ return NewPtr;
+}
+
bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
// Operands 1..: pointer, matrix, memory_layout, stride.
// OpCooperativeMatrixStoreKHR has no result.
+ // Build the element access chain before the store so its result is defined
+ // before the store that consumes it.
+ Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
- for (unsigned i = 1; i < I.getNumOperands(); ++i)
+ MIB.addUse(Ptr);
+ for (unsigned i = 2; i < I.getNumOperands(); ++i)
MIB.addUse(I.getOperand(i).getReg());
MIB.constrainAllUses(TII, TRI, RBI);
return true;
@@ -4962,7 +5002,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
case Intrinsic::spv_cooperative_matrix_load:
// result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
return selectOpWithSrcs(ResVReg, ResType, I,
- {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ {coopMatrixElementPtr(I.getOperand(2).getReg(), I),
+ I.getOperand(3).getReg(),
I.getOperand(4).getReg()},
SPIRV::OpCooperativeMatrixLoadKHR);
case Intrinsic::spv_cooperative_matrix_store:
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
new file mode 100644
index 0000000000000..57b2f14cc6979
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
@@ -0,0 +1,38 @@
+; A cooperative-matrix load/store whose source is a workgroup-shared array tile.
+; In opaque-pointer IR the source pointer is the whole Workgroup array, so the
+; selector must access-chain the array to element 0 before the cooperative-matrix
+; op, which requires a pointer to the element type.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val %}
+
+ at tile = internal addrspace(3) global [256 x half] poison, align 16
+
+; CHECK-DAG: %[[#Half:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+; The poison, non-constant Workgroup global keeps its array type (it is not
+; collapsed to a scalar pointer).
+; CHECK-DAG: %[[#Arr:]] = OpTypeArray %[[#Half]] %[[#]]
+; CHECK-DAG: %[[#ArrPtr:]] = OpTypePointer Workgroup %[[#Arr]]
+; CHECK-DAG: %[[#EltPtr:]] = OpTypePointer Workgroup %[[#Half]]
+; CHECK-DAG: %[[#Tile:]] = OpVariable %[[#ArrPtr]] Workgroup
+; CHECK-DAG: %[[#Zero:]] = OpConstant %[[#U32]] 0
+
+; The array tile is access-chained to element 0 before the load, and that scalar
+; pointer is what the cooperative-matrix load consumes.
+; CHECK: %[[#LdPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: %[[#Mat:]] = OpCooperativeMatrixLoadKHR %[[#]] %[[#LdPtr]]
+; CHECK: %[[#StPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#StPtr]] %[[#Mat]]
+
+define void @coop_matrix_workgroup_source() #0 {
+entry:
+ %m = call target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(3) @tile, i32 0, i32 16)
+ call void
+ @llvm.spv.cooperative.matrix.store(ptr addrspace(3) @tile,
+ target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0) %m, i32 0, i32 16)
+ ret void
+}
+
+attributes #0 = { "hlsl.shader"="compute" "hlsl.numthreads"="1,1,1" }
>From c004eb7f311612f5ec847ca1b3ea11899dfbc2d8 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:26:51 -0400
Subject: [PATCH 3/7] [SPIR-V] Fix OpPtrAccessChain operands in
coopMatrixElementPtr
OpPtrAccessChain takes a leading element operand that dereferences Base before
the remaining indices, unlike OpAccessChain. coopMatrixElementPtr emitted the
same single zero index for both opcodes, so on the Physical-SPIR-V path that
zero was consumed as the element and the access chain never descended into the
array. Add the element operand for the OpPtrAccessChain case; also tighten the
surrounding comments.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 30 +++++++++----------
1 file changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 5d3fdc8988f61..8c513145a97d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1611,10 +1611,8 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
-// OpCooperativeMatrixLoad/StoreKHR require the pointer to point to the element
-// type. A workgroup array tile reaches the selector as a pointer to the whole
-// array, so index it to its first element to get the scalar pointer the op
-// requires. Returns PtrReg unchanged when it is not an aggregate.
+// OpCooperativeMatrixLoad/StoreKHR need a pointer to the element type. If the
+// tile is an array, index to its first element; else return PtrReg unchanged.
Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
MachineInstr &I) const {
SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
@@ -1635,22 +1633,22 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
Register Zero = buildZerosVal(I32Type, I);
Register NewPtr = MRI->createVirtualRegister(GR.getRegClass(ElemPtrType));
GR.assignSPIRVTypeToVReg(ElemPtrType, NewPtr, *I.getParent()->getParent());
- unsigned Opcode =
- STI.isLogicalSPIRV() ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
- BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
- .addDef(NewPtr)
- .addUse(GR.getSPIRVTypeID(ElemPtrType))
- .addUse(PtrReg)
- .addUse(Zero)
- .constrainAllUses(TII, TRI, RBI);
+ bool IsLogical = STI.isLogicalSPIRV();
+ unsigned Opcode = IsLogical ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
+ auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+ .addDef(NewPtr)
+ .addUse(GR.getSPIRVTypeID(ElemPtrType))
+ .addUse(PtrReg);
+ // OpPtrAccessChain takes a leading element operand to dereference Base.
+ if (!IsLogical)
+ MIB.addUse(Zero);
+ MIB.addUse(Zero);
+ MIB.constrainAllUses(TII, TRI, RBI);
return NewPtr;
}
bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
- // Operands 1..: pointer, matrix, memory_layout, stride.
- // OpCooperativeMatrixStoreKHR has no result.
- // Build the element access chain before the store so its result is defined
- // before the store that consumes it.
+ // OpCooperativeMatrixStoreKHR ptr matrix memory_layout stride; no result.
Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
>From 85398edd009f50703867849f702fb6c7cb8976da Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:47:47 -0400
Subject: [PATCH 4/7] [SPIR-V] Remove narration comments in cooperative matrix
selection
Drop comments that restated the function, op operands, and case mappings.
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 7 -------
1 file changed, 7 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 8c513145a97d2..05f80090627b1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1611,8 +1611,6 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
-// OpCooperativeMatrixLoad/StoreKHR need a pointer to the element type. If the
-// tile is an array, index to its first element; else return PtrReg unchanged.
Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
MachineInstr &I) const {
SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
@@ -1639,7 +1637,6 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
.addDef(NewPtr)
.addUse(GR.getSPIRVTypeID(ElemPtrType))
.addUse(PtrReg);
- // OpPtrAccessChain takes a leading element operand to dereference Base.
if (!IsLogical)
MIB.addUse(Zero);
MIB.addUse(Zero);
@@ -1648,7 +1645,6 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
}
bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
- // OpCooperativeMatrixStoreKHR ptr matrix memory_layout stride; no result.
Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
@@ -4998,7 +4994,6 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
}
case Intrinsic::spv_cooperative_matrix_load:
- // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
return selectOpWithSrcs(ResVReg, ResType, I,
{coopMatrixElementPtr(I.getOperand(2).getReg(), I),
I.getOperand(3).getReg(),
@@ -5007,13 +5002,11 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
case Intrinsic::spv_cooperative_matrix_store:
return selectCoopMatrixStore(I);
case Intrinsic::spv_cooperative_matrix_muladd:
- // OpCooperativeMatrixMulAddKHR A B C; no operands literal for float matmul.
return selectOpWithSrcs(ResVReg, ResType, I,
{I.getOperand(2).getReg(), I.getOperand(3).getReg(),
I.getOperand(4).getReg()},
SPIRV::OpCooperativeMatrixMulAddKHR);
case Intrinsic::spv_cooperative_matrix_splat:
- // OpCompositeConstruct from a scalar broadcasts the accumulator.
return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
SPIRV::OpCompositeConstruct);
case Intrinsic::spv_unref_global:
>From c35156ae8dfe44010cc4f7ba09e61ff0e863d300 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:52:03 -0400
Subject: [PATCH 5/7] Apply suggestion from @MrSidims
Co-authored-by: Dmitry Sidorov <dsidorov at amd.com>
---
.../cooperative_matrix_type_vulkan.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
index e6d0aac36812f..1446c8b73f1d6 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
@@ -12,7 +12,7 @@
; CHECK-DAG: OpCapability CooperativeMatrixKHR
; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
-; CHECK-DAG: {{%[0-9]+}} = OpTypeCooperativeMatrixKHR
+; CHECK-DAG: [[#]] = OpTypeCooperativeMatrixKHR
; A by-value parameter of the opaque type forces OpTypeCooperativeMatrixKHR into the
; module (referenced by OpTypeFunction, so it is not eliminated like an unused local).
>From 7c92a27b90b7fdee4101ffc06467ec0e14133873 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:59:38 -0400
Subject: [PATCH 6/7] Clean up comments in cooperative_matrix_kernel.ll
Removed detailed comments explaining the cooperative matrix kernel functionality.
---
.../cooperative_matrix_kernel.ll | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
index d968647eecc8c..1f4434e16481a 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
@@ -1,10 +1,5 @@
; A GLCompute kernel that runs a cooperative-matrix matmul tile against
-; descriptor-bound storage buffers under the Vulkan flavor. A and B are read-only
-; StorageBuffers and C a writable StorageBuffer, each a VulkanBuffer handle
-; decorated DescriptorSet and Binding and accessed through
-; llvm.spv.resource.handlefrombinding and getpointer. The kernel loads A (use 0)
-; and B (use 1) as cooperative matrices, mul-adds into a zero accumulator C
-; (use 2), and stores C. The module must pass spirv-val --target-env vulkan1.3.
+; descriptor-bound storage buffers under the Vulkan flavor
; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
>From 95a63761b8292c0d3b6f2d041760c57fd38eaa4f Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 21:26:01 -0400
Subject: [PATCH 7/7] [SPIR-V] Use --target-env vulkan1.3 in
cooperative_matrix_workgroup_source
Match the other vulkan coop-matrix tests' spirv-val target env.
---
.../cooperative_matrix_workgroup_source.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
index 57b2f14cc6979..1d5abb4d06b36 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
@@ -4,7 +4,7 @@
; op, which requires a pointer to the element type.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
@tile = internal addrspace(3) global [256 x half] poison, align 16
More information about the llvm-commits
mailing list