[llvm] c819267 - [AArch64][CodeGen] Use interleave store for streaming compatible functions

[Caroline Concatto via llvm-commits]

Author: Caroline Concatto
Date: 2023-04-13T08:43:09Z
New Revision: c8192670ecc776e5edc5146ab2526fd0975261b5

URL: https://github.com/llvm/llvm-project/commit/c8192670ecc776e5edc5146ab2526fd0975261b5
DIFF: https://github.com/llvm/llvm-project/commit/c8192670ecc776e5edc5146ab2526fd0975261b5.diff

LOG: [AArch64][CodeGen] Use interleave store for streaming compatible functions

The previous patch, D135564, was too conservative to avoid store interleave
for streaming-compatible functions/mode.

In this patch, we allow using the interleave store but using scalable vector.

Reviewed By: david-arm, sdesmalen

Differential Revision: https://reviews.llvm.org/D147040

Added: 
    

Modified: 
    llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
    llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index da46104501237..5a227acbf6222 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14583,10 +14583,6 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
 bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
                                                   ShuffleVectorInst *SVI,
                                                   unsigned Factor) const {
-  // Skip if streaming compatible SVE is enabled, because it generates invalid
-  // code in streaming mode when SVE length is not specified.
-  if (Subtarget->forceStreamingCompatibleSVE())
-    return false;
 
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");

diff  --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
index 83894cff8259a..ae8f53f0be990 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
@@ -3,16 +3,14 @@
 
 target triple = "aarch64-unknown-linux-gnu"
 
-; Currently there is no custom lowering for vector shuffles operating on types
-; bigger than NEON. However, having no support opens us up to a code generator
-; hang when expanding BUILD_VECTOR. Here we just validate the promblematic case
-; successfully exits code generation.
 define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) #0 {
 ; CHECK-LABEL: hang_when_merging_stores_after_legalisation:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    ptrue p0.s, vl4
 ; CHECK-NEXT:    mov z0.s, s0
-; CHECK-NEXT:    stp q0, q0, [x0]
+; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    st2w { z0.s, z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <8 x i32> zeroinitializer
   %interleaved.vec = shufflevector <8 x i32> %splat, <8 x i32> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -20,6 +18,38 @@ define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) #
   ret void
 }
 
+define void @interleave_store_without_splat(ptr %a, <4 x i32> %v1, <4 x i32> %v2) #0 {
+; CHECK-LABEL: interleave_store_without_splat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <4 x i32> %v1, <4 x i32> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved = shufflevector <8 x i32> %shuffle, <8 x i32> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved, ptr %a, align 1
+  ret void
+}
+
+define void @interleave_store_legalization(ptr %a, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-LABEL: interleave_store_legalization:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #8 // =0x8
+; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    mov z5.d, z2.d
+; CHECK-NEXT:    mov z2.d, z1.d
+; CHECK-NEXT:    mov z4.d, z0.d
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    st2w { z4.s, z5.s }, p0, [x0]
+; CHECK-NEXT:    st2w { z2.s, z3.s }, p0, [x0, x8, lsl #2]
+; CHECK-NEXT:    ret
+  %interleaved.vec = shufflevector <8 x i32> %v1, <8 x i32> %v2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
+                                                                             i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i32> %interleaved.vec, ptr %a, align 4
+  ret void
+}
+
 ; Ensure we don't crash when trying to lower a shuffle via an extract
 define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) #0 {
 ; CHECK-LABEL: crash_when_lowering_extract_shuffle: