[llvm] [AArch64][ISel] Support i1 masked interleaved loads and stores (PR #209231)

[Paul Walker via llvm-commits]

================
@@ -45,6 +101,70 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
 }
 
+define { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld4_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv2i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-NEXT:    ptrue p3.d
+; CHECK-NEXT:    cmpne p0.d, p3/z, z0.d, #0
+; CHECK-NEXT:    cmpne p1.d, p3/z, z1.d, #0
+; CHECK-NEXT:    cmpne p2.d, p3/z, z2.d, #0
+; CHECK-NEXT:    cmpne p3.d, p3/z, z3.d, #0
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %wide.masked.vec = call <vscale x 8 x i1> @llvm.masked.load.nxv8i1.p0(ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask, <vscale x 8 x i1> poison)
+  %deinterleaved.vec = call { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @llvm.vector.deinterleave4.nxv8i1(<vscale x 8 x i1> %wide.masked.vec)
+  ret { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @foo_ld4_nxv4i1(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv4i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld4w { z0.s - z3.s }, p0/z, [x0]
+; CHECK-NEXT:    ptrue p3.s
+; CHECK-NEXT:    cmpne p0.s, p3/z, z0.s, #0
+; CHECK-NEXT:    cmpne p1.s, p3/z, z1.s, #0
+; CHECK-NEXT:    cmpne p2.s, p3/z, z2.s, #0
+; CHECK-NEXT:    cmpne p3.s, p3/z, z3.s, #0
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+  %wide.masked.vec = call <vscale x 16 x i1> @llvm.masked.load.nxv16i1.p0(ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask, <vscale x 16 x i1> poison)
+  %deinterleaved.vec = call { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave4.nxv16i1(<vscale x 16 x i1> %wide.masked.vec)
+  ret { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @foo_ld4_nxv8i1(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv8i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld4h { z0.h - z3.h }, p0/z, [x0]
+; CHECK-NEXT:    ptrue p3.h
+; CHECK-NEXT:    cmpne p0.h, p3/z, z0.h, #0
+; CHECK-NEXT:    cmpne p1.h, p3/z, z1.h, #0
+; CHECK-NEXT:    cmpne p2.h, p3/z, z2.h, #0
+; CHECK-NEXT:    cmpne p3.h, p3/z, z3.h, #0
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+  %wide.masked.vec = call <vscale x 32 x i1> @llvm.masked.load.nxv32i1.p0(ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask, <vscale x 32 x i1> poison)
+  %deinterleaved.vec = call { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @llvm.vector.deinterleave4.nxv32i1(<vscale x 32 x i1> %wide.masked.vec)
+  ret { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } @foo_ld4_nxv16i1(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv16i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld4b { z0.b - z3.b }, p0/z, [x0]
----------------
paulwalker-arm wrote:

Scalable boolean vector loads and stores are not well supported. Only multiples of `<vscale x 16 x i1>` work today.  Where is this coming from? At the language level there is no ABI defined for this and clang uses `i8` based vectors as the storage type for predicates.  We then spot specific patterns that can be safely lowered to predicate load/store instructions.

https://github.com/llvm/llvm-project/pull/209231