[llvm] [SPIR-V] Access-chain aggregate pointers to element 0 for cooperative matrix load/store (PR #202050)

Julian Klappenbach via llvm-commits llvm-commits at lists.llvm.org
Sun Jun 21 18:26:14 PDT 2026


https://github.com/jklappenbach updated https://github.com/llvm/llvm-project/pull/202050

>From 6bdc937544fe316d307ca57500ba76dd24f162f4 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:12 -0400
Subject: [PATCH 1/7] [SPIRV] Cooperative matrix under the Vulkan/Shader flavor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Make SPV_KHR_cooperative_matrix reachable from the Vulkan/Shader flavor (the
OpenCL __spirv_* builtin path is isShader()-gated off), mirroring the texture /
ray-query intrinsic + GlobalISel pattern:

- IntrinsicsSPIRV.td: llvm.spv.cooperative.matrix.{load,store,muladd,splat};
  the opaque matrix is carried as llvm_any_ty (overloaded per concrete shape).
- SPIRVInstructionSelector.cpp: GlobalISel selection for the four ops
  (load/muladd/splat via selectOpWithSrcs, store via selectCoopMatrixStore).
  memory-layout/stride are <id> constant operands; float MulAdd omits the
  integer-only signedness literal; splat = single-scalar OpCompositeConstruct.
- SPIRVModuleAnalysis.{cpp,h}: cooperative matrix under Shader mandates the
  Vulkan memory model (spirv-val rejects Shader + CooperativeMatrixKHR under
  GLSL450). Rather than guess up front, derive the model from the actual
  requirement: after collectReqs, if a Shader module requires CooperativeMatrixKHR
  (RequirementHandler::isCapabilityRequired, added here), upgrade GLSL450 ->
  VulkanKHR (+ VulkanMemoryModelKHR capability + SPV_KHR_vulkan_memory_model
  extension, OpMemoryModel Logical VulkanKHR), unless !spirv.MemoryModel set one
  explicitly. Non-coop shaders keep GLSL450.

The OpTypeCooperativeMatrixKHR opaque type needs no backend change — the
existing BuiltinType machinery lowers target("spirv.CooperativeMatrixKHR", elem,
scope, rows, cols, use) flavor-agnostically.
---
 llvm/include/llvm/IR/IntrinsicsSPIRV.td       | 13 +++-
 .../Target/SPIRV/SPIRVInstructionSelector.cpp | 30 +++++++
 .../cooperative_matrix_kernel.ll              | 78 +++++++++++++++++++
 .../cooperative_matrix_ops_vulkan.ll          | 53 +++++++++++++
 .../cooperative_matrix_type_vulkan.ll         | 22 ++++++
 5 files changed, 195 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 6e4cf8f7e72dc..7f8e23da39b61 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -365,5 +365,16 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
   def int_spv_unpackhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_i32_ty], [IntrNoMem]>;
   def int_spv_packhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
 
-
+  // SPV_KHR_cooperative_matrix operations under the Vulkan/Shader flavor,
+  // reached through llvm.spv intrinsics and GlobalISel selection because the
+  // __spirv_* builtin path is OpenCL-only. The matrix value is the opaque
+  // target("spirv.CooperativeMatrixKHR", ...) type, carried as llvm_any_ty.
+  def int_spv_cooperative_matrix_load
+    : Intrinsic<[llvm_any_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty]>;
+  def int_spv_cooperative_matrix_store
+    : Intrinsic<[], [llvm_anyptr_ty, llvm_any_ty, llvm_i32_ty, llvm_i32_ty]>;
+  def int_spv_cooperative_matrix_muladd
+    : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_any_ty]>;
+  def int_spv_cooperative_matrix_splat
+    : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index cd99015a61ba9..12bd09ede4ade 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -452,6 +452,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
                              MachineInstr &I) const;
   bool selectImageWriteIntrinsic(MachineInstr &I) const;
+  bool selectCoopMatrixStore(MachineInstr &I) const;
   bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
                                 MachineInstr &I) const;
   bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1609,6 +1610,17 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
   return false;
 }
 
+bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
+  // Operands 1..: pointer, matrix, memory_layout, stride.
+  // OpCooperativeMatrixStoreKHR has no result.
+  auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                     TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
+  for (unsigned i = 1; i < I.getNumOperands(); ++i)
+    MIB.addUse(I.getOperand(i).getReg());
+  MIB.constrainAllUses(TII, TRI, RBI);
+  return true;
+}
+
 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
                                                 SPIRVTypeInst ResType,
                                                 MachineInstr &I,
@@ -4947,6 +4959,24 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
       report_fatal_error("incompatible result and operand types in a bitcast");
     return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
   }
+  case Intrinsic::spv_cooperative_matrix_load:
+    // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
+    return selectOpWithSrcs(ResVReg, ResType, I,
+                            {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+                             I.getOperand(4).getReg()},
+                            SPIRV::OpCooperativeMatrixLoadKHR);
+  case Intrinsic::spv_cooperative_matrix_store:
+    return selectCoopMatrixStore(I);
+  case Intrinsic::spv_cooperative_matrix_muladd:
+    // OpCooperativeMatrixMulAddKHR A B C; no operands literal for float matmul.
+    return selectOpWithSrcs(ResVReg, ResType, I,
+                            {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+                             I.getOperand(4).getReg()},
+                            SPIRV::OpCooperativeMatrixMulAddKHR);
+  case Intrinsic::spv_cooperative_matrix_splat:
+    // OpCompositeConstruct from a scalar broadcasts the accumulator.
+    return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
+                            SPIRV::OpCompositeConstruct);
   case Intrinsic::spv_unref_global:
   case Intrinsic::spv_init_global: {
     MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
new file mode 100644
index 0000000000000..d968647eecc8c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
@@ -0,0 +1,78 @@
+; A GLCompute kernel that runs a cooperative-matrix matmul tile against
+; descriptor-bound storage buffers under the Vulkan flavor. A and B are read-only
+; StorageBuffers and C a writable StorageBuffer, each a VulkanBuffer handle
+; decorated DescriptorSet and Binding and accessed through
+; llvm.spv.resource.handlefrombinding and getpointer. The kernel loads A (use 0)
+; and B (use 1) as cooperative matrices, mul-adds into a zero accumulator C
+; (use 2), and stores C. The module must pass spirv-val --target-env vulkan1.3.
+
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpEntryPoint GLCompute %[[#entry:]] "main"
+; CHECK-DAG: OpDecorate %[[#A:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#A]] Binding 0
+; CHECK-DAG: OpDecorate %[[#B:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#B]] Binding 1
+; CHECK-DAG: OpDecorate %[[#C:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#C]] Binding 2
+; CHECK: %[[#MA:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MB:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MC0:]] = OpCompositeConstruct
+; CHECK: %[[#MC:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#MA]] %[[#MB]] %[[#MC0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#MC]]
+
+ at .str.a = private unnamed_addr constant [2 x i8] c"a\00", align 1
+ at .str.b = private unnamed_addr constant [2 x i8] c"b\00", align 1
+ at .str.c = private unnamed_addr constant [2 x i8] c"c\00", align 1
+
+define void @main() local_unnamed_addr #0 {
+entry:
+  ; A: read-only StorageBuffer, binding 0 -> MatrixA (use 0)
+  %ha = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+      @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+          i32 0, i32 0, i32 1, i32 0, ptr nonnull @.str.a)
+  %pa = tail call ptr addrspace(11)
+      @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+          target("spirv.VulkanBuffer", [0 x float], 12, 0) %ha, i32 0)
+  %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+      @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pa, i32 0, i32 16)
+
+  ; B: read-only StorageBuffer, binding 1 -> MatrixB (use 1)
+  %hb = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+      @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+          i32 0, i32 1, i32 1, i32 0, ptr nonnull @.str.b)
+  %pb = tail call ptr addrspace(11)
+      @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+          target("spirv.VulkanBuffer", [0 x float], 12, 0) %hb, i32 0)
+  %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+      @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pb, i32 0, i32 16)
+
+  ; C = A*B + 0  (accumulator, use 2), stored to writable StorageBuffer binding 2
+  %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+      @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+  %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+      @llvm.spv.cooperative.matrix.muladd(
+          target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+          target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+          target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+  %hc = tail call target("spirv.VulkanBuffer", [0 x float], 12, 1)
+      @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_1t(
+          i32 0, i32 2, i32 1, i32 0, ptr nonnull @.str.c)
+  %pc = tail call ptr addrspace(11)
+      @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_1t(
+          target("spirv.VulkanBuffer", [0 x float], 12, 1) %hc, i32 0)
+  call void @llvm.spv.cooperative.matrix.store(
+          ptr addrspace(11) %pc,
+          target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+          i32 0, i32 16)
+  ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
new file mode 100644
index 0000000000000..365fa2cd9a404
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
@@ -0,0 +1,53 @@
+; The cooperative-matrix operations (load, store, mul-add, splat-construct) under
+; the Vulkan/Shader flavor, reached via llvm.spv.cooperative.matrix.* intrinsics
+; and GlobalISel selection because the OpenCL __spirv_CooperativeMatrix* builtin
+; path is OpenCL-only.
+;
+; A single matmul tile fragment: C = A * B + 0. A is MatrixA (use 0), B is MatrixB
+; (use 1), C the accumulator (use 2); all 16x16 f32, Subgroup scope (3). Memory
+; layout 0 = RowMajorKHR, stride 16.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; The module sets the Vulkan memory model explicitly via spirv.MemoryModel
+; metadata (spirv-val rejects Shader + CooperativeMatrixKHR under GLSL450).
+; CHECK-DAG: OpCapability VulkanMemoryModelKHR
+; CHECK-DAG: OpExtension "SPV_KHR_vulkan_memory_model"
+; CHECK-DAG: OpMemoryModel Logical VulkanKHR
+; CHECK-DAG: %[[#F32:]] = OpTypeFloat 32
+; The three matrix types (A use 0, B use 1, C accumulator use 2) all lower to
+; OpTypeCooperativeMatrixKHR over the f32 component type.
+; CHECK-DAG: OpTypeCooperativeMatrixKHR %[[#F32]]
+; Data flow: mul-add consumes both loaded operands plus the splat-constructed
+; accumulator; the store consumes the mul-add result.
+; CHECK: %[[#A:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#B:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#C0:]] = OpCompositeConstruct
+; CHECK: %[[#C:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#A]] %[[#B]] %[[#C0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#C]]
+
+define spir_func void @matmul_tile(ptr %pa, ptr %pb, ptr %pc) {
+entry:
+  %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+       @llvm.spv.cooperative.matrix.load(ptr %pa, i32 0, i32 16)
+  %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+       @llvm.spv.cooperative.matrix.load(ptr %pb, i32 0, i32 16)
+  %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+        @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+  %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+       @llvm.spv.cooperative.matrix.muladd(
+         target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+         target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+         target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+  call void @llvm.spv.cooperative.matrix.store(
+         ptr %pc,
+         target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+         i32 0, i32 16)
+  ret void
+}
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
new file mode 100644
index 0000000000000..e6d0aac36812f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
@@ -0,0 +1,22 @@
+; Check that the opaque CooperativeMatrix type under the Vulkan/Shader flavor,
+; target("spirv.CooperativeMatrixKHR", elem, scope, rows, cols, use), lowers to
+; OpTypeCooperativeMatrixKHR, gated behind the CooperativeMatrixKHR capability and
+; the SPV_KHR_cooperative_matrix extension, and that it errors cleanly without it.
+; This is a text-emission check only; the type is forced via a by-value parameter,
+; so it is not run through spirv-val here.
+
+; RUN: not llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-ERROR: LLVM ERROR: OpTypeCooperativeMatrixKHR type requires the following SPIR-V extension: SPV_KHR_cooperative_matrix
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: {{%[0-9]+}} = OpTypeCooperativeMatrixKHR
+
+; A by-value parameter of the opaque type forces OpTypeCooperativeMatrixKHR into the
+; module (referenced by OpTypeFunction, so it is not eliminated like an unused local).
+define spir_func void @use_coopmat(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %m) {
+entry:
+  ret void
+}

>From 76913939000c3548bbb9280c680e72e8738462ee Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:52 -0400
Subject: [PATCH 2/7] [SPIR-V] Access-chain aggregate pointers to element 0 for
 cooperative matrix load/store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

OpCooperativeMatrixLoadKHR / OpCooperativeMatrixStoreKHR require the Pointer to
point to a scalar or vector — the tile's element type. When the source is a
workgroup-shared array tile, however, the pointer reaching the selector is a
pointer to the whole `[N x T]` array: in opaque-pointer IR `&arr[0]` is the same
SSA value as `&arr`, and a zero-index element GEP is simplified back to the array
base during SPIRVEmitIntrinsics. The cooperative-matrix op was then emitted with
an array pointer, which spirv-val rejects ("Pointer's Type must be a scalar or
vector type").

Index such a pointer to its first element with an OpAccessChain before the
cooperative-matrix op, so it receives the required scalar pointer. Index 0 selects
the same address the MemoryLayout/Stride operands then walk from, so semantics are
unchanged. Pointers that are already element-typed (e.g. a StorageBuffer access
chain, or a dynamic-offset Workgroup access chain) are not aggregates and pass
through untouched.

Verified with llc + spirv-val on a Workgroup-tile cooperative-matrix load, and a
full LDS-staged GEMM (CoopStage copy -> Barrier -> load(Shared) -> mma) computing
bit-exact results on RADV / gfx1151.
---
 .../Target/SPIRV/SPIRVInstructionSelector.cpp | 45 ++++++++++++++++++-
 .../cooperative_matrix_workgroup_source.ll    | 38 ++++++++++++++++
 2 files changed, 81 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 12bd09ede4ade..5d3fdc8988f61 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -453,6 +453,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
                              MachineInstr &I) const;
   bool selectImageWriteIntrinsic(MachineInstr &I) const;
   bool selectCoopMatrixStore(MachineInstr &I) const;
+  Register coopMatrixElementPtr(Register PtrReg, MachineInstr &I) const;
   bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
                                 MachineInstr &I) const;
   bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1610,12 +1611,51 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
   return false;
 }
 
+// OpCooperativeMatrixLoad/StoreKHR require the pointer to point to the element
+// type. A workgroup array tile reaches the selector as a pointer to the whole
+// array, so index it to its first element to get the scalar pointer the op
+// requires. Returns PtrReg unchanged when it is not an aggregate.
+Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
+                                                        MachineInstr &I) const {
+  SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
+  if (!PtrType)
+    return PtrReg;
+  SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
+  if (!PointeeType || PointeeType->getOpcode() != SPIRV::OpTypeArray)
+    return PtrReg;
+  SPIRVTypeInst ElemType =
+      GR.getSPIRVTypeForVReg(PointeeType->getOperand(1).getReg());
+  if (!ElemType)
+    return PtrReg;
+  SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(PtrReg);
+  MachineIRBuilder MIRBuilder(I);
+  SPIRVTypeInst ElemPtrType =
+      GR.getOrCreateSPIRVPointerType(ElemType, MIRBuilder, SC);
+  SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
+  Register Zero = buildZerosVal(I32Type, I);
+  Register NewPtr = MRI->createVirtualRegister(GR.getRegClass(ElemPtrType));
+  GR.assignSPIRVTypeToVReg(ElemPtrType, NewPtr, *I.getParent()->getParent());
+  unsigned Opcode =
+      STI.isLogicalSPIRV() ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
+  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+      .addDef(NewPtr)
+      .addUse(GR.getSPIRVTypeID(ElemPtrType))
+      .addUse(PtrReg)
+      .addUse(Zero)
+      .constrainAllUses(TII, TRI, RBI);
+  return NewPtr;
+}
+
 bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
   // Operands 1..: pointer, matrix, memory_layout, stride.
   // OpCooperativeMatrixStoreKHR has no result.
+  // Build the element access chain before the store so its result is defined
+  // before the store that consumes it.
+  Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
-  for (unsigned i = 1; i < I.getNumOperands(); ++i)
+  MIB.addUse(Ptr);
+  for (unsigned i = 2; i < I.getNumOperands(); ++i)
     MIB.addUse(I.getOperand(i).getReg());
   MIB.constrainAllUses(TII, TRI, RBI);
   return true;
@@ -4962,7 +5002,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
   case Intrinsic::spv_cooperative_matrix_load:
     // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
     return selectOpWithSrcs(ResVReg, ResType, I,
-                            {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+                            {coopMatrixElementPtr(I.getOperand(2).getReg(), I),
+                             I.getOperand(3).getReg(),
                              I.getOperand(4).getReg()},
                             SPIRV::OpCooperativeMatrixLoadKHR);
   case Intrinsic::spv_cooperative_matrix_store:
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
new file mode 100644
index 0000000000000..57b2f14cc6979
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
@@ -0,0 +1,38 @@
+; A cooperative-matrix load/store whose source is a workgroup-shared array tile.
+; In opaque-pointer IR the source pointer is the whole Workgroup array, so the
+; selector must access-chain the array to element 0 before the cooperative-matrix
+; op, which requires a pointer to the element type.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val %}
+
+ at tile = internal addrspace(3) global [256 x half] poison, align 16
+
+; CHECK-DAG: %[[#Half:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+; The poison, non-constant Workgroup global keeps its array type (it is not
+; collapsed to a scalar pointer).
+; CHECK-DAG: %[[#Arr:]] = OpTypeArray %[[#Half]] %[[#]]
+; CHECK-DAG: %[[#ArrPtr:]] = OpTypePointer Workgroup %[[#Arr]]
+; CHECK-DAG: %[[#EltPtr:]] = OpTypePointer Workgroup %[[#Half]]
+; CHECK-DAG: %[[#Tile:]] = OpVariable %[[#ArrPtr]] Workgroup
+; CHECK-DAG: %[[#Zero:]] = OpConstant %[[#U32]] 0
+
+; The array tile is access-chained to element 0 before the load, and that scalar
+; pointer is what the cooperative-matrix load consumes.
+; CHECK: %[[#LdPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: %[[#Mat:]] = OpCooperativeMatrixLoadKHR %[[#]] %[[#LdPtr]]
+; CHECK: %[[#StPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#StPtr]] %[[#Mat]]
+
+define void @coop_matrix_workgroup_source() #0 {
+entry:
+  %m = call target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0)
+       @llvm.spv.cooperative.matrix.load(ptr addrspace(3) @tile, i32 0, i32 16)
+  call void
+       @llvm.spv.cooperative.matrix.store(ptr addrspace(3) @tile,
+       target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0) %m, i32 0, i32 16)
+  ret void
+}
+
+attributes #0 = { "hlsl.shader"="compute" "hlsl.numthreads"="1,1,1" }

>From c004eb7f311612f5ec847ca1b3ea11899dfbc2d8 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:26:51 -0400
Subject: [PATCH 3/7] [SPIR-V] Fix OpPtrAccessChain operands in
 coopMatrixElementPtr

OpPtrAccessChain takes a leading element operand that dereferences Base before
the remaining indices, unlike OpAccessChain. coopMatrixElementPtr emitted the
same single zero index for both opcodes, so on the Physical-SPIR-V path that
zero was consumed as the element and the access chain never descended into the
array. Add the element operand for the OpPtrAccessChain case; also tighten the
surrounding comments.
---
 .../Target/SPIRV/SPIRVInstructionSelector.cpp | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 5d3fdc8988f61..8c513145a97d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1611,10 +1611,8 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
   return false;
 }
 
-// OpCooperativeMatrixLoad/StoreKHR require the pointer to point to the element
-// type. A workgroup array tile reaches the selector as a pointer to the whole
-// array, so index it to its first element to get the scalar pointer the op
-// requires. Returns PtrReg unchanged when it is not an aggregate.
+// OpCooperativeMatrixLoad/StoreKHR need a pointer to the element type. If the
+// tile is an array, index to its first element; else return PtrReg unchanged.
 Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
                                                         MachineInstr &I) const {
   SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
@@ -1635,22 +1633,22 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
   Register Zero = buildZerosVal(I32Type, I);
   Register NewPtr = MRI->createVirtualRegister(GR.getRegClass(ElemPtrType));
   GR.assignSPIRVTypeToVReg(ElemPtrType, NewPtr, *I.getParent()->getParent());
-  unsigned Opcode =
-      STI.isLogicalSPIRV() ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
-      .addDef(NewPtr)
-      .addUse(GR.getSPIRVTypeID(ElemPtrType))
-      .addUse(PtrReg)
-      .addUse(Zero)
-      .constrainAllUses(TII, TRI, RBI);
+  bool IsLogical = STI.isLogicalSPIRV();
+  unsigned Opcode = IsLogical ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
+  auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+                 .addDef(NewPtr)
+                 .addUse(GR.getSPIRVTypeID(ElemPtrType))
+                 .addUse(PtrReg);
+  // OpPtrAccessChain takes a leading element operand to dereference Base.
+  if (!IsLogical)
+    MIB.addUse(Zero);
+  MIB.addUse(Zero);
+  MIB.constrainAllUses(TII, TRI, RBI);
   return NewPtr;
 }
 
 bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
-  // Operands 1..: pointer, matrix, memory_layout, stride.
-  // OpCooperativeMatrixStoreKHR has no result.
-  // Build the element access chain before the store so its result is defined
-  // before the store that consumes it.
+  // OpCooperativeMatrixStoreKHR ptr matrix memory_layout stride; no result.
   Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(SPIRV::OpCooperativeMatrixStoreKHR));

>From 85398edd009f50703867849f702fb6c7cb8976da Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:47:47 -0400
Subject: [PATCH 4/7] [SPIR-V] Remove narration comments in cooperative matrix
 selection

Drop comments that restated the function, op operands, and case mappings.
---
 llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 8c513145a97d2..05f80090627b1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1611,8 +1611,6 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
   return false;
 }
 
-// OpCooperativeMatrixLoad/StoreKHR need a pointer to the element type. If the
-// tile is an array, index to its first element; else return PtrReg unchanged.
 Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
                                                         MachineInstr &I) const {
   SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
@@ -1639,7 +1637,6 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
                  .addDef(NewPtr)
                  .addUse(GR.getSPIRVTypeID(ElemPtrType))
                  .addUse(PtrReg);
-  // OpPtrAccessChain takes a leading element operand to dereference Base.
   if (!IsLogical)
     MIB.addUse(Zero);
   MIB.addUse(Zero);
@@ -1648,7 +1645,6 @@ Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
 }
 
 bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
-  // OpCooperativeMatrixStoreKHR ptr matrix memory_layout stride; no result.
   Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
@@ -4998,7 +4994,6 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
   }
   case Intrinsic::spv_cooperative_matrix_load:
-    // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
     return selectOpWithSrcs(ResVReg, ResType, I,
                             {coopMatrixElementPtr(I.getOperand(2).getReg(), I),
                              I.getOperand(3).getReg(),
@@ -5007,13 +5002,11 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
   case Intrinsic::spv_cooperative_matrix_store:
     return selectCoopMatrixStore(I);
   case Intrinsic::spv_cooperative_matrix_muladd:
-    // OpCooperativeMatrixMulAddKHR A B C; no operands literal for float matmul.
     return selectOpWithSrcs(ResVReg, ResType, I,
                             {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
                              I.getOperand(4).getReg()},
                             SPIRV::OpCooperativeMatrixMulAddKHR);
   case Intrinsic::spv_cooperative_matrix_splat:
-    // OpCompositeConstruct from a scalar broadcasts the accumulator.
     return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
                             SPIRV::OpCompositeConstruct);
   case Intrinsic::spv_unref_global:

>From c35156ae8dfe44010cc4f7ba09e61ff0e863d300 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:52:03 -0400
Subject: [PATCH 5/7] Apply suggestion from @MrSidims

Co-authored-by: Dmitry Sidorov <dsidorov at amd.com>
---
 .../cooperative_matrix_type_vulkan.ll                           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
index e6d0aac36812f..1446c8b73f1d6 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
@@ -12,7 +12,7 @@
 
 ; CHECK-DAG: OpCapability CooperativeMatrixKHR
 ; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
-; CHECK-DAG: {{%[0-9]+}} = OpTypeCooperativeMatrixKHR
+; CHECK-DAG: [[#]] = OpTypeCooperativeMatrixKHR
 
 ; A by-value parameter of the opaque type forces OpTypeCooperativeMatrixKHR into the
 ; module (referenced by OpTypeFunction, so it is not eliminated like an unused local).

>From 7c92a27b90b7fdee4101ffc06467ec0e14133873 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 20:59:38 -0400
Subject: [PATCH 6/7] Clean up comments in cooperative_matrix_kernel.ll

Removed detailed comments explaining the cooperative matrix kernel functionality.
---
 .../cooperative_matrix_kernel.ll                           | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
index d968647eecc8c..1f4434e16481a 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
@@ -1,10 +1,5 @@
 ; A GLCompute kernel that runs a cooperative-matrix matmul tile against
-; descriptor-bound storage buffers under the Vulkan flavor. A and B are read-only
-; StorageBuffers and C a writable StorageBuffer, each a VulkanBuffer handle
-; decorated DescriptorSet and Binding and accessed through
-; llvm.spv.resource.handlefrombinding and getpointer. The kernel loads A (use 0)
-; and B (use 1) as cooperative matrices, mul-adds into a zero accumulator C
-; (use 2), and stores C. The module must pass spirv-val --target-env vulkan1.3.
+; descriptor-bound storage buffers under the Vulkan flavor
 
 ; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}

>From 95a63761b8292c0d3b6f2d041760c57fd38eaa4f Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sun, 21 Jun 2026 21:26:01 -0400
Subject: [PATCH 7/7] [SPIR-V] Use --target-env vulkan1.3 in
 cooperative_matrix_workgroup_source

Match the other vulkan coop-matrix tests' spirv-val target env.
---
 .../cooperative_matrix_workgroup_source.ll                      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
index 57b2f14cc6979..1d5abb4d06b36 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
@@ -4,7 +4,7 @@
 ; op, which requires a pointer to the element type.
 
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
 
 @tile = internal addrspace(3) global [256 x half] poison, align 16
 



More information about the llvm-commits mailing list