[llvm] [AArch64] Lower fixed-length interleaved stores with SVE (PR #213692)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 07:53:18 PDT 2026
================
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK-COMPAT,CHECK-COMPAT-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK-COMPAT,CHECK-COMPAT-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IADISABLED
+
+define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-COMPAT-LABEL: store_factor3:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl8
+; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor3:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
+; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
+; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl8
+; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
+ <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-COMPAT-LABEL: store_factor4:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z19.d, z6.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl4
+; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor4:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
+; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
+; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
+; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl4
+; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
+ <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved, ptr %ptr, align 8
+ ret void
+}
+
+
+define void @store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) #0 {
+; CHECK-COMPAT-LABEL: store_factor2_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: adrp x8, .LCPI2_0
+; CHECK-COMPAT-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
+; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-COMPAT-NEXT: ld1w { z1.s }, p1/z, [x8]
+; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-COMPAT-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor2_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: adrp x8, .LCPI2_0
+; CHECK-STREAMING-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
+; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: ld1w { z1.s }, p1/z, [x8]
+; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-STREAMING-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
+ <4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) #0 {
+; CHECK-COMPAT-LABEL: store_factor3_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor3_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <12 x i32> @llvm.vector.interleave3.v12i32(
+ <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3) #0 {
+; CHECK-COMPAT-LABEL: store_factor4_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor4_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i64> @llvm.vector.interleave4.v8i64(
+ <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3)
+ store <8 x i64> %interleaved, ptr %ptr, align 8
+ ret void
+}
+
+define void @masked_store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor3:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: uunpklo z0.h, z6.b
+; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-COMPAT-NEXT: uunpklo z0.s, z0.h
+; CHECK-COMPAT-NEXT: lsl z0.s, z0.s, #31
+; CHECK-COMPAT-NEXT: asr z0.s, z0.s, #31
+; CHECK-COMPAT-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor3:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: uunpklo z6.h, z6.b
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
+; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
+; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
+; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: uunpklo z6.s, z6.h
+; CHECK-STREAMING-NEXT: lsl z6.s, z6.s, #31
+; CHECK-STREAMING-NEXT: asr z6.s, z6.s, #31
+; CHECK-STREAMING-NEXT: cmpne p0.s, p1/z, z6.s, #0
+; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
+ <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(
+ <8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
+ call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved, ptr %ptr,
+ i32 4, <24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor4:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z19.d, z6.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: ptrue p1.d, vl4
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: ldr d2, [sp]
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
+; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-COMPAT-NEXT: uunpklo z0.d, z2.s
+; CHECK-COMPAT-NEXT: lsl z0.d, z0.d, #63
+; CHECK-COMPAT-NEXT: asr z0.d, z0.d, #63
+; CHECK-COMPAT-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor4:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ldr d16, [sp]
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: ptrue p1.d, vl4
+; CHECK-STREAMING-NEXT: uunpklo z16.s, z16.h
+; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
+; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
+; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
+; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
+; CHECK-STREAMING-NEXT: uunpklo z16.d, z16.s
+; CHECK-STREAMING-NEXT: lsl z16.d, z16.d, #63
+; CHECK-STREAMING-NEXT: asr z7.d, z16.d, #63
+; CHECK-STREAMING-NEXT: cmpne p0.d, p1/z, z7.d, #0
+; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
+ <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(
+ <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved, ptr %ptr,
+ i32 8, <16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor3_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 def $z3
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: uunpklo z3.s, z3.h
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: lsl z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: asr z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor3_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: uunpklo z3.s, z3.h
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: lsl z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: asr z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <12 x i32> @llvm.vector.interleave3.v12i32(
+ <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ %interleaved.mask = call <12 x i1> @llvm.vector.interleave3.v12i1(
+ <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v12i32.p0(<12 x i32> %interleaved, ptr %ptr,
+ i32 4, <12 x i1> %interleaved.mask)
+ ret void
+}
+
+attributes #0 = { vscale_range(2,16) }
----------------
kamleshbhalui wrote:
removed
https://github.com/llvm/llvm-project/pull/213692
More information about the llvm-commits
mailing list