[llvm] 2e3f4ef - [SPIRV] Legalize Cast Instructions for Matrix Types (#212619)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 18:00:02 PDT 2026
Author: Farzon Lotfi
Date: 2026-07-30T20:59:57-04:00
New Revision: 2e3f4ef5855e07f4bd2b0a1270e6101beceb1f8c
URL: https://github.com/llvm/llvm-project/commit/2e3f4ef5855e07f4bd2b0a1270e6101beceb1f8c
DIFF: https://github.com/llvm/llvm-project/commit/2e3f4ef5855e07f4bd2b0a1270e6101beceb1f8c.diff
LOG: [SPIRV] Legalize Cast Instructions for Matrix Types (#212619)
fixes #186864
This is a more limited fix. that doesn't require us to change the whole
legalization strategy for SPIR-V.
To get the last fix to work it looks like I would have to have changed
every global opcode to no longer use power of 2 based legalization.
This change with power of 2 fixed the transpose case in the offload
testsuite when we do the conversion of bools to i32s before we call
transpose.
There remains an issue when a i1 makes it into a transpose intrinsic.
That said I can only trigger that case via llvmir and not HLSL so will
move on for now.
Added:
llvm/test/CodeGen/SPIRV/legalization/matrix-wide-vector-shader.ll
Modified:
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 21586e242404a..6e073b6fa5d41 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -257,7 +257,10 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
// Illegal G_UNMERGE_VALUES instructions should be handled
// during the combine phase.
getActionDefinitionsBuilder(G_BUILD_VECTOR)
- .legalIf(vectorElementCountIsLessThanOrEqualTo(0, MaxVectorSize));
+ .legalIf(vectorElementCountIsLessThanOrEqualTo(0, MaxVectorSize))
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),
+ LegalizeMutations::changeElementCountTo(
+ 0, ElementCount::getFixed(MaxVectorSize)));
// When entering the legalizer, there should be no G_BITCAST instructions.
// They should all be calls to the `spv_bitcast` intrinsic. The call to
@@ -351,14 +354,20 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
.legalForCartesianProduct(allIntScalarsAndVectors)
.legalIf(extendedScalarsAndVectorsProduct);
- // Extensions.
getActionDefinitionsBuilder({G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
.legalForCartesianProduct(allScalarsAndVectors)
- .legalIf(extendedScalarsAndVectorsProduct);
+ .legalIf(extendedScalarsAndVectorsProduct)
+ .moreElementsToNextPow2(0)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),
+ LegalizeMutations::changeElementCountTo(
+ 0, ElementCount::getFixed(MaxVectorSize)));
- // Lower G_SEXT_INREG to the canonical shl/ashr pair, which map to
- // OpShiftLeftLogical + OpShiftRightArithmetic.
- getActionDefinitionsBuilder(G_SEXT_INREG).lower();
+ getActionDefinitionsBuilder(G_SEXT_INREG)
+ .moreElementsToNextPow2(0)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),
+ LegalizeMutations::changeElementCountTo(
+ 0, ElementCount::getFixed(MaxVectorSize)))
+ .lower();
getActionDefinitionsBuilder(G_PHI)
.legalFor(allPtrsScalarsAndVectors)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index 2b33dabde4612..e2b1a1b37d8ca 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -49,7 +49,12 @@ static SPIRVTypeInst deduceIntTypeFromResult(Register ResVReg,
MachineIRBuilder &MIB,
SPIRVGlobalRegistry *GR) {
const LLT &Ty = MIB.getMRI()->getType(ResVReg);
- return GR->getOrCreateSPIRVIntegerType(Ty.getScalarSizeInBits(), MIB);
+ SPIRVTypeInst ScalarType =
+ GR->getOrCreateSPIRVIntegerType(Ty.getScalarSizeInBits(), MIB);
+ if (Ty.isVector())
+ return GR->getOrCreateSPIRVVectorType(ScalarType, Ty.getNumElements(), MIB,
+ false);
+ return ScalarType;
}
static SPIRVTypeInst deduceTypeFromSingleOperand(MachineInstr *I,
diff --git a/llvm/test/CodeGen/SPIRV/legalization/matrix-wide-vector-shader.ll b/llvm/test/CodeGen/SPIRV/legalization/matrix-wide-vector-shader.ll
new file mode 100644
index 0000000000000..7f9591fe27755
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/legalization/matrix-wide-vector-shader.ll
@@ -0,0 +1,222 @@
+; RUN: llc -O0 -mtriple=spirv1.6-unknown-vulkan1.3 %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-unknown-vulkan1.3 %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+ at Ints9 = internal addrspace(10) global [9 x i32] poison
+ at Bools9 = internal addrspace(10) global [9 x i32] poison
+ at Ints12 = internal addrspace(10) global [12 x i32] poison
+ at Bools12 = internal addrspace(10) global [12 x i32] poison
+ at Ints16 = internal addrspace(10) global [16 x i32] poison
+ at Bools16 = internal addrspace(10) global [16 x i32] poison
+
+; CHECK-DAG: %[[Int32:[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: %[[Vec4Int32:[0-9]+]] = OpTypeVector %[[Int32]] 4
+
+; No vector wider than 4 lanes is ever materialized for shader targets.
+; CHECK-NOT: OpTypeVector %[[Int32]] 8
+; CHECK-NOT: OpTypeVector %[[Int32]] 9
+; CHECK-NOT: OpTypeVector %[[Int32]] 12
+; CHECK-NOT: OpTypeVector %[[Int32]] 16
+
+;--- G_LOAD/G_STORE: always scalarized, even for the pow2-sized 4x4 case ---
+
+; CHECK-LABEL: ; -- Begin function copy_bool3x3
+; CHECK-COUNT-9: OpLoad %[[Int32]]
+; CHECK-COUNT-9: OpStore
+define internal void @copy_bool3x3() {
+ %m = load <9 x i32>, ptr addrspace(10) @Ints9
+ store <9 x i32> %m, ptr addrspace(10) @Bools9
+ ret void
+}
+
+; Stands in for both bool3x4 and bool4x3 (both flatten to 12 elements).
+; CHECK-LABEL: ; -- Begin function copy_bool_12elem
+; CHECK-COUNT-12: OpLoad %[[Int32]]
+; CHECK-COUNT-12: OpStore
+define internal void @copy_bool_12elem() {
+ %m = load <12 x i32>, ptr addrspace(10) @Ints12
+ store <12 x i32> %m, ptr addrspace(10) @Bools12
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function copy_bool4x4
+; CHECK-COUNT-16: OpLoad %[[Int32]]
+; CHECK-COUNT-16: OpStore
+define internal void @copy_bool4x4() {
+ %m = load <16 x i32>, ptr addrspace(10) @Ints16
+ store <16 x i32> %m, ptr addrspace(10) @Bools16
+ ret void
+}
+
+;--- G_TRUNC + G_ZEXT: trunc-to-i1 then zext folds to (x & 1) ---
+;
+; The wide vector is rebuilt as N <= 4-lane chunks (all N OpCompositeConstruct
+; first), then each chunk is masked with its own OpBitwiseAnd (all N masks
+; after). Each mask's chunk operand is named and checked explicitly, in the
+; same order codegen actually produces them, so the dataflow is tracked
+; rather than counted blindly.
+
+; CHECK-LABEL: ; -- Begin function bool3x3_zext
+; CHECK: %[[C0:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C1:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C2:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C0]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C1]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C2]]
+; CHECK-NOT: OpBitwiseAnd %[[Vec4Int32]]
+define internal void @bool3x3_zext() {
+ %m = load <9 x i32>, ptr addrspace(10) @Ints9
+ %bits = trunc <9 x i32> %m to <9 x i1>
+ %ext = zext <9 x i1> %bits to <9 x i32>
+ store <9 x i32> %ext, ptr addrspace(10) @Bools9
+ ret void
+}
+
+; Stands in for both bool3x4 and bool4x3.
+; CHECK-LABEL: ; -- Begin function bool_12elem_zext
+; CHECK: %[[C0:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C1:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C2:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C0]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C1]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C2]]
+; CHECK-NOT: OpBitwiseAnd %[[Vec4Int32]]
+define internal void @bool_12elem_zext() {
+ %m = load <12 x i32>, ptr addrspace(10) @Ints12
+ %bits = trunc <12 x i32> %m to <12 x i1>
+ %ext = zext <12 x i1> %bits to <12 x i32>
+ store <12 x i32> %ext, ptr addrspace(10) @Bools12
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function bool4x4_zext
+; CHECK: %[[C0:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C1:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C2:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C3:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C0]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C1]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C2]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C3]]
+; CHECK-NOT: OpBitwiseAnd %[[Vec4Int32]]
+define internal void @bool4x4_zext() {
+ %m = load <16 x i32>, ptr addrspace(10) @Ints16
+ %bits = trunc <16 x i32> %m to <16 x i1>
+ %ext = zext <16 x i1> %bits to <16 x i32>
+ store <16 x i32> %ext, ptr addrspace(10) @Bools16
+ ret void
+}
+
+;--- G_TRUNC + G_SEXT: trunc-to-i1 then sext canonicalizes to G_SEXT_INREG ---
+;
+; This target lowers G_SEXT_INREG to a shl/ashr pair by (bitwidth - 1), one
+; pair per chunk, with each pair emitted back-to-back (unlike the zext masks
+; above). Every chunk's shift-left-feeding-shift-right pair is named and
+; checked explicitly below.
+
+; CHECK-LABEL: ; -- Begin function bool3x3_sext
+; CHECK: %[[Shl0:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl0]]
+; CHECK: %[[Shl1:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl1]]
+; CHECK: %[[Shl2:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl2]]
+; CHECK-NOT: OpShiftLeftLogical %[[Vec4Int32]]
+define internal void @bool3x3_sext() {
+ %m = load <9 x i32>, ptr addrspace(10) @Ints9
+ %bits = trunc <9 x i32> %m to <9 x i1>
+ %ext = sext <9 x i1> %bits to <9 x i32>
+ store <9 x i32> %ext, ptr addrspace(10) @Bools9
+ ret void
+}
+
+; Stands in for both bool3x4 and bool4x3.
+; CHECK-LABEL: ; -- Begin function bool_12elem_sext
+; CHECK: %[[Shl0:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl0]]
+; CHECK: %[[Shl1:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl1]]
+; CHECK: %[[Shl2:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl2]]
+; CHECK-NOT: OpShiftLeftLogical %[[Vec4Int32]]
+define internal void @bool_12elem_sext() {
+ %m = load <12 x i32>, ptr addrspace(10) @Ints12
+ %bits = trunc <12 x i32> %m to <12 x i1>
+ %ext = sext <12 x i1> %bits to <12 x i32>
+ store <12 x i32> %ext, ptr addrspace(10) @Bools12
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function bool4x4_sext
+; CHECK: %[[Shl0:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl0]]
+; CHECK: %[[Shl1:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl1]]
+; CHECK: %[[Shl2:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl2]]
+; CHECK: %[[Shl3:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl3]]
+; CHECK-NOT: OpShiftLeftLogical %[[Vec4Int32]]
+define internal void @bool4x4_sext() {
+ %m = load <16 x i32>, ptr addrspace(10) @Ints16
+ %bits = trunc <16 x i32> %m to <16 x i1>
+ %ext = sext <16 x i1> %bits to <16 x i32>
+ store <16 x i32> %ext, ptr addrspace(10) @Bools16
+ ret void
+}
+
+;--- Non-bool G_TRUNC/G_ZEXT/G_SEXT: same chunking, wider element type ---
+;
+; trunc-to-i16 then zext/sext-to-i32 folds the same way the i1 case does
+; (mask-with-0xFFFF for zext, shl/ashr-by-16 for sext), so this checks the
+; chunk splitting is independent of the truncated width, not just i1.
+
+; CHECK-LABEL: ; -- Begin function narrow_12elem_zext
+; CHECK: %[[C0:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C1:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %[[C2:[0-9]+]] = OpCompositeConstruct %[[Vec4Int32]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C0]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C1]]
+; CHECK: %{{[0-9]+}} = OpBitwiseAnd %[[Vec4Int32]] %[[C2]]
+; CHECK-NOT: OpBitwiseAnd %[[Vec4Int32]]
+define internal void @narrow_12elem_zext() {
+ %m = load <12 x i32>, ptr addrspace(10) @Ints12
+ %narrow = trunc <12 x i32> %m to <12 x i16>
+ %wide = zext <12 x i16> %narrow to <12 x i32>
+ store <12 x i32> %wide, ptr addrspace(10) @Bools12
+ ret void
+}
+
+; CHECK-LABEL: ; -- Begin function narrow_16elem_sext
+; CHECK: %[[Shl0:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl0]]
+; CHECK: %[[Shl1:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl1]]
+; CHECK: %[[Shl2:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl2]]
+; CHECK: %[[Shl3:[0-9]+]] = OpShiftLeftLogical %[[Vec4Int32]]
+; CHECK-NEXT: %{{[0-9]+}} = OpShiftRightArithmetic %[[Vec4Int32]] %[[Shl3]]
+; CHECK-NOT: OpShiftLeftLogical %[[Vec4Int32]]
+define internal void @narrow_16elem_sext() {
+ %m = load <16 x i32>, ptr addrspace(10) @Ints16
+ %narrow = trunc <16 x i32> %m to <16 x i16>
+ %wide = sext <16 x i16> %narrow to <16 x i32>
+ store <16 x i32> %wide, ptr addrspace(10) @Bools16
+ ret void
+}
+
+define void @main() #0 {
+ call void @copy_bool3x3()
+ call void @copy_bool_12elem()
+ call void @copy_bool4x4()
+ call void @bool3x3_zext()
+ call void @bool_12elem_zext()
+ call void @bool4x4_zext()
+ call void @bool3x3_sext()
+ call void @bool_12elem_sext()
+ call void @bool4x4_sext()
+ call void @narrow_12elem_zext()
+ call void @narrow_16elem_sext()
+ ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
More information about the llvm-commits
mailing list