[llvm] [SPIR-V] Access-chain aggregate pointers to element 0 for cooperative matrix load/store (PR #202050)
Julian Klappenbach via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 18:07:58 PDT 2026
https://github.com/jklappenbach updated https://github.com/llvm/llvm-project/pull/202050
>From 8520d15d8254edca37eb37c3d886c2d6c0312e77 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:12 -0400
Subject: [PATCH 1/3] [SPIRV] Cooperative matrix under the Vulkan/Shader flavor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Make SPV_KHR_cooperative_matrix reachable from the Vulkan/Shader flavor (the
OpenCL __spirv_* builtin path is isShader()-gated off), mirroring the texture /
ray-query intrinsic + GlobalISel pattern:
- IntrinsicsSPIRV.td: llvm.spv.cooperative.matrix.{load,store,muladd,splat};
the opaque matrix is carried as llvm_any_ty (overloaded per concrete shape).
- SPIRVInstructionSelector.cpp: GlobalISel selection for the four ops
(load/muladd/splat via selectOpWithSrcs, store via selectCoopMatrixStore).
memory-layout/stride are <id> constant operands; float MulAdd omits the
integer-only signedness literal; splat = single-scalar OpCompositeConstruct.
- SPIRVModuleAnalysis.{cpp,h}: cooperative matrix under Shader mandates the
Vulkan memory model (spirv-val rejects Shader + CooperativeMatrixKHR under
GLSL450). Rather than guess up front, derive the model from the actual
requirement: after collectReqs, if a Shader module requires CooperativeMatrixKHR
(RequirementHandler::isCapabilityRequired, added here), upgrade GLSL450 ->
VulkanKHR (+ VulkanMemoryModelKHR capability + SPV_KHR_vulkan_memory_model
extension, OpMemoryModel Logical VulkanKHR), unless !spirv.MemoryModel set one
explicitly. Non-coop shaders keep GLSL450.
The OpTypeCooperativeMatrixKHR opaque type needs no backend change — the
existing BuiltinType machinery lowers target("spirv.CooperativeMatrixKHR", elem,
scope, rows, cols, use) flavor-agnostically.
---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 13 +++-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 30 +++++++
.../cooperative_matrix_kernel.ll | 78 +++++++++++++++++++
.../cooperative_matrix_ops_vulkan.ll | 53 +++++++++++++
.../cooperative_matrix_type_vulkan.ll | 22 ++++++
5 files changed, 195 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 1e758f9f63f49..2c465ceb3bf0d 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -360,5 +360,16 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]
def int_spv_unpackhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_i32_ty], [IntrNoMem]>;
def int_spv_packhalf2x16 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
-
+ // SPV_KHR_cooperative_matrix operations under the Vulkan/Shader flavor,
+ // reached through llvm.spv intrinsics and GlobalISel selection because the
+ // __spirv_* builtin path is OpenCL-only. The matrix value is the opaque
+ // target("spirv.CooperativeMatrixKHR", ...) type, carried as llvm_any_ty.
+ def int_spv_cooperative_matrix_load
+ : Intrinsic<[llvm_any_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty]>;
+ def int_spv_cooperative_matrix_store
+ : Intrinsic<[], [llvm_anyptr_ty, llvm_any_ty, llvm_i32_ty, llvm_i32_ty]>;
+ def int_spv_cooperative_matrix_muladd
+ : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_any_ty]>;
+ def int_spv_cooperative_matrix_splat
+ : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index c3f21fe025bd5..8f1e098389a84 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -444,6 +444,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectImageWriteIntrinsic(MachineInstr &I) const;
+ bool selectCoopMatrixStore(MachineInstr &I) const;
bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1592,6 +1593,17 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
+bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
+ // Operands 1..: pointer, matrix, memory_layout, stride.
+ // OpCooperativeMatrixStoreKHR has no result.
+ auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+ TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
+ for (unsigned i = 1; i < I.getNumOperands(); ++i)
+ MIB.addUse(I.getOperand(i).getReg());
+ MIB.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I,
@@ -4708,6 +4720,24 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
report_fatal_error("incompatible result and operand types in a bitcast");
return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast);
}
+ case Intrinsic::spv_cooperative_matrix_load:
+ // result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
+ return selectOpWithSrcs(ResVReg, ResType, I,
+ {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ I.getOperand(4).getReg()},
+ SPIRV::OpCooperativeMatrixLoadKHR);
+ case Intrinsic::spv_cooperative_matrix_store:
+ return selectCoopMatrixStore(I);
+ case Intrinsic::spv_cooperative_matrix_muladd:
+ // OpCooperativeMatrixMulAddKHR A B C; no operands literal for float matmul.
+ return selectOpWithSrcs(ResVReg, ResType, I,
+ {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ I.getOperand(4).getReg()},
+ SPIRV::OpCooperativeMatrixMulAddKHR);
+ case Intrinsic::spv_cooperative_matrix_splat:
+ // OpCompositeConstruct from a scalar broadcasts the accumulator.
+ return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(2).getReg()},
+ SPIRV::OpCompositeConstruct);
case Intrinsic::spv_unref_global:
case Intrinsic::spv_init_global: {
MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
new file mode 100644
index 0000000000000..d968647eecc8c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_kernel.ll
@@ -0,0 +1,78 @@
+; A GLCompute kernel that runs a cooperative-matrix matmul tile against
+; descriptor-bound storage buffers under the Vulkan flavor. A and B are read-only
+; StorageBuffers and C a writable StorageBuffer, each a VulkanBuffer handle
+; decorated DescriptorSet and Binding and accessed through
+; llvm.spv.resource.handlefrombinding and getpointer. The kernel loads A (use 0)
+; and B (use 1) as cooperative matrices, mul-adds into a zero accumulator C
+; (use 2), and stores C. The module must pass spirv-val --target-env vulkan1.3.
+
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: OpEntryPoint GLCompute %[[#entry:]] "main"
+; CHECK-DAG: OpDecorate %[[#A:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#A]] Binding 0
+; CHECK-DAG: OpDecorate %[[#B:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#B]] Binding 1
+; CHECK-DAG: OpDecorate %[[#C:]] DescriptorSet 0
+; CHECK-DAG: OpDecorate %[[#C]] Binding 2
+; CHECK: %[[#MA:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MB:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#MC0:]] = OpCompositeConstruct
+; CHECK: %[[#MC:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#MA]] %[[#MB]] %[[#MC0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#MC]]
+
+ at .str.a = private unnamed_addr constant [2 x i8] c"a\00", align 1
+ at .str.b = private unnamed_addr constant [2 x i8] c"b\00", align 1
+ at .str.c = private unnamed_addr constant [2 x i8] c"c\00", align 1
+
+define void @main() local_unnamed_addr #0 {
+entry:
+ ; A: read-only StorageBuffer, binding 0 -> MatrixA (use 0)
+ %ha = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+ i32 0, i32 0, i32 1, i32 0, ptr nonnull @.str.a)
+ %pa = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 0) %ha, i32 0)
+ %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pa, i32 0, i32 16)
+
+ ; B: read-only StorageBuffer, binding 1 -> MatrixB (use 1)
+ %hb = tail call target("spirv.VulkanBuffer", [0 x float], 12, 0)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_0t(
+ i32 0, i32 1, i32 1, i32 0, ptr nonnull @.str.b)
+ %pb = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_0t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 0) %hb, i32 0)
+ %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(11) %pb, i32 0, i32 16)
+
+ ; C = A*B + 0 (accumulator, use 2), stored to writable StorageBuffer binding 2
+ %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+ %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.muladd(
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+ %hc = tail call target("spirv.VulkanBuffer", [0 x float], 12, 1)
+ @llvm.spv.resource.handlefrombinding.tspirv.VulkanBuffer_a0f32_12_1t(
+ i32 0, i32 2, i32 1, i32 0, ptr nonnull @.str.c)
+ %pc = tail call ptr addrspace(11)
+ @llvm.spv.resource.getpointer.p11.tspirv.VulkanBuffer_a0f32_12_1t(
+ target("spirv.VulkanBuffer", [0 x float], 12, 1) %hc, i32 0)
+ call void @llvm.spv.cooperative.matrix.store(
+ ptr addrspace(11) %pc,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+ i32 0, i32 16)
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
new file mode 100644
index 0000000000000..365fa2cd9a404
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_ops_vulkan.ll
@@ -0,0 +1,53 @@
+; The cooperative-matrix operations (load, store, mul-add, splat-construct) under
+; the Vulkan/Shader flavor, reached via llvm.spv.cooperative.matrix.* intrinsics
+; and GlobalISel selection because the OpenCL __spirv_CooperativeMatrix* builtin
+; path is OpenCL-only.
+;
+; A single matmul tile fragment: C = A * B + 0. A is MatrixA (use 0), B is MatrixB
+; (use 1), C the accumulator (use 2); all 16x16 f32, Subgroup scope (3). Memory
+; layout 0 = RowMajorKHR, stride 16.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; The module sets the Vulkan memory model explicitly via spirv.MemoryModel
+; metadata (spirv-val rejects Shader + CooperativeMatrixKHR under GLSL450).
+; CHECK-DAG: OpCapability VulkanMemoryModelKHR
+; CHECK-DAG: OpExtension "SPV_KHR_vulkan_memory_model"
+; CHECK-DAG: OpMemoryModel Logical VulkanKHR
+; CHECK-DAG: %[[#F32:]] = OpTypeFloat 32
+; The three matrix types (A use 0, B use 1, C accumulator use 2) all lower to
+; OpTypeCooperativeMatrixKHR over the f32 component type.
+; CHECK-DAG: OpTypeCooperativeMatrixKHR %[[#F32]]
+; Data flow: mul-add consumes both loaded operands plus the splat-constructed
+; accumulator; the store consumes the mul-add result.
+; CHECK: %[[#A:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#B:]] = OpCooperativeMatrixLoadKHR
+; CHECK: %[[#C0:]] = OpCompositeConstruct
+; CHECK: %[[#C:]] = OpCooperativeMatrixMulAddKHR %[[#]] %[[#A]] %[[#B]] %[[#C0]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#]] %[[#C]]
+
+define spir_func void @matmul_tile(ptr %pa, ptr %pb, ptr %pc) {
+entry:
+ %a = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr %pa, i32 0, i32 16)
+ %b = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1)
+ @llvm.spv.cooperative.matrix.load(ptr %pb, i32 0, i32 16)
+ %c0 = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.splat(float 0.000000e+00)
+ %c = call target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2)
+ @llvm.spv.cooperative.matrix.muladd(
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 0) %a,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 1) %b,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c0)
+ call void @llvm.spv.cooperative.matrix.store(
+ ptr %pc,
+ target("spirv.CooperativeMatrixKHR", float, 3, 16, 16, 2) %c,
+ i32 0, i32 16)
+ ret void
+}
+
+; AddressingModel=Logical (0), MemoryModel=VulkanKHR (3)
+!spirv.MemoryModel = !{!0}
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
new file mode 100644
index 0000000000000..e6d0aac36812f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_type_vulkan.ll
@@ -0,0 +1,22 @@
+; Check that the opaque CooperativeMatrix type under the Vulkan/Shader flavor,
+; target("spirv.CooperativeMatrixKHR", elem, scope, rows, cols, use), lowers to
+; OpTypeCooperativeMatrixKHR, gated behind the CooperativeMatrixKHR capability and
+; the SPV_KHR_cooperative_matrix extension, and that it errors cleanly without it.
+; This is a text-emission check only; the type is forced via a by-value parameter,
+; so it is not run through spirv-val here.
+
+; RUN: not llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+
+; CHECK-ERROR: LLVM ERROR: OpTypeCooperativeMatrixKHR type requires the following SPIR-V extension: SPV_KHR_cooperative_matrix
+
+; CHECK-DAG: OpCapability CooperativeMatrixKHR
+; CHECK-DAG: OpExtension "SPV_KHR_cooperative_matrix"
+; CHECK-DAG: {{%[0-9]+}} = OpTypeCooperativeMatrixKHR
+
+; A by-value parameter of the opaque type forces OpTypeCooperativeMatrixKHR into the
+; module (referenced by OpTypeFunction, so it is not eliminated like an unused local).
+define spir_func void @use_coopmat(target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) %m) {
+entry:
+ ret void
+}
>From d4beba3a3521411ce1807e5ba9511750e2157952 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:10 -0400
Subject: [PATCH 2/3] [SPIR-V] Deduce pointee type for all global variables,
not only initialized ones
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
SPIRVEmitIntrinsics::processGlobalValue only recorded a global variable's
element type (deduceElementTypeHelper) when hasInitializer() was true. For an
undef-initialized, non-constant aggregate global — e.g. a Workgroup `[N x T]`
shared tile (`@g = internal addrspace(3) global [N x T] undef`) — hasInitializer
returns false, so the variable's type was never recorded from its declared value
type. Its pointee type was then inferred lazily from a flat element-typed GEP use
(`getelementptr T, ptr @g, %i`) as the scalar `T`.
With the variable typed as a scalar pointer, the array-to-pointer-decay rewrite
in visitGetElementPtrInst (which prepends a 0 index so a Logical-SPIR-V access
chain can index the array) is skipped, because its DeducedPointeeTy is not an
ArrayType. The GEP stays a byte-offset pointer-arithmetic form Logical SPIR-V
cannot express, and the dynamic index is dropped: every invocation accesses
element 0. A Workgroup array indexed by a loop variable thus collapsed to a
scalar variable, silently corrupting workgroup-shared reductions/staging on
Vulkan (the emitted module passes spirv-val but computes wrong results).
A global variable's pointee type is concrete and authoritative, so record it for
every global in processGlobalValue (which runs before the use-based forward
pass). The deduction result is still ignored at the call site — TypedPointerType
isn't expressible in general LLVM IR; it is stored in the Global Registry.
(cherry picked from commit 2849c532820328544bee3ea3d8acd25289d3f457)
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 9 ++---
.../type-deduce-global-array-poison.ll | 33 +++++++++++++++++++
2 files changed, 38 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-array-poison.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 97fa49d8836fb..6a99bea0db1bc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2553,12 +2553,13 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, CurrF))
return;
+ // Record the pointee type for every global, not only initialized ones, so an
+ // undef non-constant aggregate global is not later collapsed to its element
+ // type. Result is ignored; the type is stored in the Global Registry.
+ deduceElementTypeHelper(&GV, false);
+
Constant *Init = nullptr;
if (hasInitializer(&GV)) {
- // Deduce element type and store results in Global Registry.
- // Result is ignored, because TypedPointerType is not supported
- // by llvm IR general logic.
- deduceElementTypeHelper(&GV, false);
Init = GV.getInitializer();
Value *InitOp = Init;
if (isa<UndefValue>(Init) && Init->getType()->isAggregateType()) {
diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-array-poison.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-array-poison.ll
new file mode 100644
index 0000000000000..4b5097a9e9c9e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-global-array-poison.ll
@@ -0,0 +1,33 @@
+; A global variable keeps its concrete array pointee type even when its
+; initializer is poison and it is non-constant, so a dynamic index is lowered as
+; an OpAccessChain rather than dropped (which would make every invocation access
+; element 0).
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
+
+ at tile = internal addrspace(3) global [64 x i32] poison, align 16
+
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Arr:]] = OpTypeArray %[[#U32]] %[[#]]
+; CHECK-DAG: %[[#ArrPtr:]] = OpTypePointer Workgroup %[[#Arr]]
+; CHECK-DAG: %[[#EltPtr:]] = OpTypePointer Workgroup %[[#U32]]
+; CHECK-DAG: %[[#Tile:]] = OpVariable %[[#ArrPtr]] Workgroup
+
+; The dynamic index survives as an OpAccessChain into the array tile and the
+; variable is not collapsed to a scalar pointer.
+; CHECK: %[[#Id:]] = OpCompositeExtract %[[#U32]] %[[#]] 0
+; CHECK: %[[#Ptr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Id]]
+; CHECK: OpStore %[[#Ptr]] %[[#]]
+
+define void @store_dynamic_index() #0 {
+entry:
+ %id = call i32 @llvm.spv.thread.id.in.group.i32(i32 0)
+ %p = getelementptr i32, ptr addrspace(3) @tile, i32 %id
+ store i32 42, ptr addrspace(3) %p, align 4
+ ret void
+}
+
+declare i32 @llvm.spv.thread.id.in.group.i32(i32)
+
+attributes #0 = { "hlsl.shader"="compute" "hlsl.numthreads"="64,1,1" }
>From 6cb0af14637641e73ce96629c6322c488d54b4e9 Mon Sep 17 00:00:00 2001
From: Julian Klappenbach <julian at twilight.digital>
Date: Sat, 6 Jun 2026 11:14:52 -0400
Subject: [PATCH 3/3] [SPIR-V] Access-chain aggregate pointers to element 0 for
cooperative matrix load/store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
OpCooperativeMatrixLoadKHR / OpCooperativeMatrixStoreKHR require the Pointer to
point to a scalar or vector — the tile's element type. When the source is a
workgroup-shared array tile, however, the pointer reaching the selector is a
pointer to the whole `[N x T]` array: in opaque-pointer IR `&arr[0]` is the same
SSA value as `&arr`, and a zero-index element GEP is simplified back to the array
base during SPIRVEmitIntrinsics. The cooperative-matrix op was then emitted with
an array pointer, which spirv-val rejects ("Pointer's Type must be a scalar or
vector type").
Index such a pointer to its first element with an OpAccessChain before the
cooperative-matrix op, so it receives the required scalar pointer. Index 0 selects
the same address the MemoryLayout/Stride operands then walk from, so semantics are
unchanged. Pointers that are already element-typed (e.g. a StorageBuffer access
chain, or a dynamic-offset Workgroup access chain) are not aggregates and pass
through untouched.
Verified with llc + spirv-val on a Workgroup-tile cooperative-matrix load, and a
full LDS-staged GEMM (CoopStage copy -> Barrier -> load(Shared) -> mma) computing
bit-exact results on RADV / gfx1151.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 45 ++++++++++++++++++-
.../cooperative_matrix_workgroup_source.ll | 38 ++++++++++++++++
2 files changed, 81 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 8f1e098389a84..81e8388e778e3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -445,6 +445,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
MachineInstr &I) const;
bool selectImageWriteIntrinsic(MachineInstr &I) const;
bool selectCoopMatrixStore(MachineInstr &I) const;
+ Register coopMatrixElementPtr(Register PtrReg, MachineInstr &I) const;
bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType,
@@ -1593,12 +1594,51 @@ bool SPIRVInstructionSelector::selectSincos(Register ResVReg,
return false;
}
+// OpCooperativeMatrixLoad/StoreKHR require the pointer to point to the element
+// type. A workgroup array tile reaches the selector as a pointer to the whole
+// array, so index it to its first element to get the scalar pointer the op
+// requires. Returns PtrReg unchanged when it is not an aggregate.
+Register SPIRVInstructionSelector::coopMatrixElementPtr(Register PtrReg,
+ MachineInstr &I) const {
+ SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrReg);
+ if (!PtrType)
+ return PtrReg;
+ SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
+ if (!PointeeType || PointeeType->getOpcode() != SPIRV::OpTypeArray)
+ return PtrReg;
+ SPIRVTypeInst ElemType =
+ GR.getSPIRVTypeForVReg(PointeeType->getOperand(1).getReg());
+ if (!ElemType)
+ return PtrReg;
+ SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(PtrReg);
+ MachineIRBuilder MIRBuilder(I);
+ SPIRVTypeInst ElemPtrType =
+ GR.getOrCreateSPIRVPointerType(ElemType, MIRBuilder, SC);
+ SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII);
+ Register Zero = buildZerosVal(I32Type, I);
+ Register NewPtr = MRI->createVirtualRegister(GR.getRegClass(ElemPtrType));
+ GR.assignSPIRVTypeToVReg(ElemPtrType, NewPtr, *I.getParent()->getParent());
+ unsigned Opcode =
+ STI.isLogicalSPIRV() ? SPIRV::OpAccessChain : SPIRV::OpPtrAccessChain;
+ BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+ .addDef(NewPtr)
+ .addUse(GR.getSPIRVTypeID(ElemPtrType))
+ .addUse(PtrReg)
+ .addUse(Zero)
+ .constrainAllUses(TII, TRI, RBI);
+ return NewPtr;
+}
+
bool SPIRVInstructionSelector::selectCoopMatrixStore(MachineInstr &I) const {
// Operands 1..: pointer, matrix, memory_layout, stride.
// OpCooperativeMatrixStoreKHR has no result.
+ // Build the element access chain before the store so its result is defined
+ // before the store that consumes it.
+ Register Ptr = coopMatrixElementPtr(I.getOperand(1).getReg(), I);
auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
TII.get(SPIRV::OpCooperativeMatrixStoreKHR));
- for (unsigned i = 1; i < I.getNumOperands(); ++i)
+ MIB.addUse(Ptr);
+ for (unsigned i = 2; i < I.getNumOperands(); ++i)
MIB.addUse(I.getOperand(i).getReg());
MIB.constrainAllUses(TII, TRI, RBI);
return true;
@@ -4723,7 +4763,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
case Intrinsic::spv_cooperative_matrix_load:
// result = OpCooperativeMatrixLoadKHR ptr memory_layout stride
return selectOpWithSrcs(ResVReg, ResType, I,
- {I.getOperand(2).getReg(), I.getOperand(3).getReg(),
+ {coopMatrixElementPtr(I.getOperand(2).getReg(), I),
+ I.getOperand(3).getReg(),
I.getOperand(4).getReg()},
SPIRV::OpCooperativeMatrixLoadKHR);
case Intrinsic::spv_cooperative_matrix_store:
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
new file mode 100644
index 0000000000000..fc31188e1ffae
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_cooperative_matrix/cooperative_matrix_workgroup_source.ll
@@ -0,0 +1,38 @@
+; A cooperative-matrix load/store whose source is a workgroup-shared array tile.
+; In opaque-pointer IR the source pointer is the whole Workgroup array, so the
+; selector must access-chain the array to element 0 before the cooperative-matrix
+; op, which requires a pointer to the element type.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_vulkan_memory_model %s -o - -filetype=obj | spirv-val %}
+
+ at tile = internal addrspace(3) global [256 x half] undef, align 16
+
+; CHECK-DAG: %[[#Half:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
+; The undef, non-constant Workgroup global keeps its array type (it is not
+; collapsed to a scalar pointer).
+; CHECK-DAG: %[[#Arr:]] = OpTypeArray %[[#Half]] %[[#]]
+; CHECK-DAG: %[[#ArrPtr:]] = OpTypePointer Workgroup %[[#Arr]]
+; CHECK-DAG: %[[#EltPtr:]] = OpTypePointer Workgroup %[[#Half]]
+; CHECK-DAG: %[[#Tile:]] = OpVariable %[[#ArrPtr]] Workgroup
+; CHECK-DAG: %[[#Zero:]] = OpConstant %[[#U32]] 0
+
+; The array tile is access-chained to element 0 before the load, and that scalar
+; pointer is what the cooperative-matrix load consumes.
+; CHECK: %[[#LdPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: %[[#Mat:]] = OpCooperativeMatrixLoadKHR %[[#]] %[[#LdPtr]]
+; CHECK: %[[#StPtr:]] = OpAccessChain %[[#EltPtr]] %[[#Tile]] %[[#Zero]]
+; CHECK: OpCooperativeMatrixStoreKHR %[[#StPtr]] %[[#Mat]]
+
+define void @coop_matrix_workgroup_source() #0 {
+entry:
+ %m = call target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0)
+ @llvm.spv.cooperative.matrix.load(ptr addrspace(3) @tile, i32 0, i32 16)
+ call void
+ @llvm.spv.cooperative.matrix.store(ptr addrspace(3) @tile,
+ target("spirv.CooperativeMatrixKHR", half, 3, 16, 16, 0) %m, i32 0, i32 16)
+ ret void
+}
+
+attributes #0 = { "hlsl.shader"="compute" "hlsl.numthreads"="1,1,1" }
More information about the llvm-commits
mailing list