[llvm] [AArch64] Lower fixed-length interleaved stores with SVE (PR #213692)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 06:39:28 PDT 2026
================
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+
+define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
+; CHECK-LABEL: store_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
+ store <32 x i16> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-LABEL: store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-LABEL: store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z0.h, z6.b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: asr z0.s, z0.s, #31
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(<8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
+ call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved.vec, ptr %ptr, i32 4, <24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ldr d2, [sp]
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: uunpklo z0.d, z2.s
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: asr z0.d, z0.d, #63
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved.vec, ptr %ptr, i32 8, <16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_128bit_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ %interleaved.mask = call <12 x i1> @llvm.vector.interleave3.v12i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v12i32.p0(<12 x i32> %interleaved.vec, ptr %ptr, i32 4, <12 x i1> %interleaved.mask)
+ ret void
+}
+
+attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
----------------
paulwalker-arm wrote:
Please put the `vscale_range(2,2)` inline with the function because it is then easier to understand what the test is verifying.
https://github.com/llvm/llvm-project/pull/213692
More information about the llvm-commits
mailing list