[llvm] [AArch64] Support wide interleaved store combine (PR #217856)

Kamlesh Kumar via llvm-commits llvm-commits at lists.llvm.org
Fri Aug 21 02:15:12 PDT 2026


https://github.com/kamleshbhalui created https://github.com/llvm/llvm-project/pull/217856

Splitting fixed-length interleaves whose inputs are wider than a NEON register into a optimal sequence of stN to prevent type legalization from splitting the interleave itself, leaving ZIPs and plain stores that can no longer be recognized as an interleaved store.

>From 2d98230b614bd816078a2085bf143d29159ee909 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 20 Aug 2026 19:36:41 +0000
Subject: [PATCH 1/2] added the test

---
 .../AArch64/vector-interleave-store.ll        | 638 ++++++++++++++++++
 1 file changed, 638 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index d3985991af531..9732dfc246e70 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
 ; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
 ; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+
 define void @aarch64_vector_interleave_idx_st2(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1) {
 ; CHECK-IAENABLED-LABEL: aarch64_vector_interleave_idx_st2:
 ; CHECK-IAENABLED:       // %bb.0: // %entry
@@ -66,3 +67,640 @@ entry:
   store <16 x float> %interleave, ptr %gep1, align 16
   ret void
 }
+
+define void @wide_vector_interleave_st2(ptr %input0, ptr %input1, ptr %output) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st2:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q2, [x0]
+; CHECK-IAENABLED-NEXT:    ldp q1, q3, [x1]
+; CHECK-IAENABLED-NEXT:    st2 { v0.4s, v1.4s }, [x2], #32
+; CHECK-IAENABLED-NEXT:    st2 { v2.4s, v3.4s }, [x2]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st2:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    ldp q3, q0, [x1]
+; CHECK-IADISABLED-NEXT:    ldp q2, q1, [x0]
+; CHECK-IADISABLED-NEXT:    zip1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT:    zip2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT:    zip1 v1.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT:    zip2 v2.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT:    stp q4, q0, [x2, #32]
+; CHECK-IADISABLED-NEXT:    stp q1, q2, [x2]
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <8 x i32>, ptr %input0, align 4
+  %v1 = load <8 x i32>, ptr %input1, align 4
+  %interleaved.vec = tail call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
+  store <16 x i32> %interleaved.vec, ptr %output, align 8
+  ret void
+}
+
+define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-IAENABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
+; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT:    mov x8, sp
+; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x8]
+; CHECK-IADISABLED-NEXT:    add x8, sp, #48
+; CHECK-IADISABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x8]
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT:    add sp, sp, #96
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <8 x i32>, ptr %input0, align 4
+  %v1 = load <8 x i32>, ptr %input1, align 4
+  %v2 = load <8 x i32>, ptr %input2, align 4
+  %interleave = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+  store <24 x i32> %interleave, ptr %output, align 4
+  ret void
+}
+
+define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT:    st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT:    zip2 v16.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT:    zip1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT:    zip2 v2.4s, v5.4s, v7.4s
+; CHECK-IADISABLED-NEXT:    zip2 v17.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT:    zip1 v1.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT:    zip2 v3.4s, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT:    zip1 v5.4s, v5.4s, v7.4s
+; CHECK-IADISABLED-NEXT:    zip1 v4.4s, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT:    zip1 v18.4s, v17.4s, v16.4s
+; CHECK-IADISABLED-NEXT:    zip2 v16.4s, v17.4s, v16.4s
+; CHECK-IADISABLED-NEXT:    zip1 v6.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT:    zip2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT:    zip1 v1.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT:    zip2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT:    zip1 v3.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT:    zip2 v4.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <8 x i32>, ptr %input0, align 4
+  %v1 = load <8 x i32>, ptr %input1, align 4
+  %v2 = load <8 x i32>, ptr %input2, align 4
+  %v3 = load <8 x i32>, ptr %input3, align 4
+  %interleave = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3)
+  store <32 x i32> %interleave, ptr %output, align 4
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i8:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
+; CHECK-IAENABLED-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i8:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
+; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT:    mov x8, sp
+; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x8]
+; CHECK-IADISABLED-NEXT:    add x8, sp, #48
+; CHECK-IADISABLED-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x8]
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT:    add sp, sp, #96
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <32 x i8>, ptr %input0, align 1
+  %v1 = load <32 x i8>, ptr %input1, align 1
+  %v2 = load <32 x i8>, ptr %input2, align 1
+  %interleave = call <96 x i8> @llvm.vector.interleave3.v96i8(<32 x i8> %v0, <32 x i8> %v1, <32 x i8> %v2)
+  store <96 x i8> %interleave, ptr %output, align 1
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i8:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT:    st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+; CHECK-IAENABLED-NEXT:    st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i8:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT:    zip2 v16.16b, v0.16b, v2.16b
+; CHECK-IADISABLED-NEXT:    zip1 v0.16b, v0.16b, v2.16b
+; CHECK-IADISABLED-NEXT:    zip2 v2.16b, v5.16b, v7.16b
+; CHECK-IADISABLED-NEXT:    zip2 v17.16b, v3.16b, v1.16b
+; CHECK-IADISABLED-NEXT:    zip1 v1.16b, v3.16b, v1.16b
+; CHECK-IADISABLED-NEXT:    zip2 v3.16b, v4.16b, v6.16b
+; CHECK-IADISABLED-NEXT:    zip1 v5.16b, v5.16b, v7.16b
+; CHECK-IADISABLED-NEXT:    zip1 v4.16b, v4.16b, v6.16b
+; CHECK-IADISABLED-NEXT:    zip1 v18.16b, v17.16b, v16.16b
+; CHECK-IADISABLED-NEXT:    zip2 v16.16b, v17.16b, v16.16b
+; CHECK-IADISABLED-NEXT:    zip1 v6.16b, v1.16b, v0.16b
+; CHECK-IADISABLED-NEXT:    zip2 v0.16b, v1.16b, v0.16b
+; CHECK-IADISABLED-NEXT:    zip1 v1.16b, v3.16b, v2.16b
+; CHECK-IADISABLED-NEXT:    zip2 v2.16b, v3.16b, v2.16b
+; CHECK-IADISABLED-NEXT:    zip1 v3.16b, v4.16b, v5.16b
+; CHECK-IADISABLED-NEXT:    zip2 v4.16b, v4.16b, v5.16b
+; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <32 x i8>, ptr %input0, align 1
+  %v1 = load <32 x i8>, ptr %input1, align 1
+  %v2 = load <32 x i8>, ptr %input2, align 1
+  %v3 = load <32 x i8>, ptr %input3, align 1
+  %interleave = call <128 x i8> @llvm.vector.interleave4.v128i8(<32 x i8> %v0, <32 x i8> %v1, <32 x i8> %v2, <32 x i8> %v3)
+  store <128 x i8> %interleave, ptr %output, align 1
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i16:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
+; CHECK-IAENABLED-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i16:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
+; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT:    mov x8, sp
+; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x8]
+; CHECK-IADISABLED-NEXT:    add x8, sp, #48
+; CHECK-IADISABLED-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x8]
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT:    add sp, sp, #96
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <16 x i16>, ptr %input0, align 2
+  %v1 = load <16 x i16>, ptr %input1, align 2
+  %v2 = load <16 x i16>, ptr %input2, align 2
+  %interleave = call <48 x i16> @llvm.vector.interleave3.v48i16(<16 x i16> %v0, <16 x i16> %v1, <16 x i16> %v2)
+  store <48 x i16> %interleave, ptr %output, align 2
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i16:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT:    st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
+; CHECK-IAENABLED-NEXT:    st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i16:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT:    zip2 v16.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT:    zip1 v0.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT:    zip2 v2.8h, v5.8h, v7.8h
+; CHECK-IADISABLED-NEXT:    zip2 v17.8h, v3.8h, v1.8h
+; CHECK-IADISABLED-NEXT:    zip1 v1.8h, v3.8h, v1.8h
+; CHECK-IADISABLED-NEXT:    zip2 v3.8h, v4.8h, v6.8h
+; CHECK-IADISABLED-NEXT:    zip1 v5.8h, v5.8h, v7.8h
+; CHECK-IADISABLED-NEXT:    zip1 v4.8h, v4.8h, v6.8h
+; CHECK-IADISABLED-NEXT:    zip1 v18.8h, v17.8h, v16.8h
+; CHECK-IADISABLED-NEXT:    zip2 v16.8h, v17.8h, v16.8h
+; CHECK-IADISABLED-NEXT:    zip1 v6.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT:    zip2 v0.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT:    zip1 v1.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT:    zip2 v2.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT:    zip1 v3.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT:    zip2 v4.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <16 x i16>, ptr %input0, align 2
+  %v1 = load <16 x i16>, ptr %input1, align 2
+  %v2 = load <16 x i16>, ptr %input2, align 2
+  %v3 = load <16 x i16>, ptr %input3, align 2
+  %interleave = call <64 x i16> @llvm.vector.interleave4.v64i16(<16 x i16> %v0, <16 x i16> %v1, <16 x i16> %v2, <16 x i16> %v3)
+  store <64 x i16> %interleave, ptr %output, align 2
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i64:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; CHECK-IAENABLED-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i64:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
+; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT:    mov x8, sp
+; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x8]
+; CHECK-IADISABLED-NEXT:    add x8, sp, #48
+; CHECK-IADISABLED-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x8]
+; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT:    add sp, sp, #96
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <4 x i64>, ptr %input0, align 8
+  %v1 = load <4 x i64>, ptr %input1, align 8
+  %v2 = load <4 x i64>, ptr %input2, align 8
+  %interleave = call <12 x i64> @llvm.vector.interleave3.v12i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2)
+  store <12 x i64> %interleave, ptr %output, align 8
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i64:
+; CHECK-IAENABLED:       // %bb.0:
+; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT:    st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
+; CHECK-IAENABLED-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
+; CHECK-IAENABLED-NEXT:    ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i64:
+; CHECK-IADISABLED:       // %bb.0:
+; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT:    zip2 v16.2d, v0.2d, v2.2d
+; CHECK-IADISABLED-NEXT:    zip1 v0.2d, v0.2d, v2.2d
+; CHECK-IADISABLED-NEXT:    zip2 v2.2d, v5.2d, v7.2d
+; CHECK-IADISABLED-NEXT:    zip2 v17.2d, v3.2d, v1.2d
+; CHECK-IADISABLED-NEXT:    zip1 v1.2d, v3.2d, v1.2d
+; CHECK-IADISABLED-NEXT:    zip2 v3.2d, v4.2d, v6.2d
+; CHECK-IADISABLED-NEXT:    zip1 v5.2d, v5.2d, v7.2d
+; CHECK-IADISABLED-NEXT:    zip1 v4.2d, v4.2d, v6.2d
+; CHECK-IADISABLED-NEXT:    zip1 v18.2d, v17.2d, v16.2d
+; CHECK-IADISABLED-NEXT:    zip2 v16.2d, v17.2d, v16.2d
+; CHECK-IADISABLED-NEXT:    zip1 v6.2d, v1.2d, v0.2d
+; CHECK-IADISABLED-NEXT:    zip2 v0.2d, v1.2d, v0.2d
+; CHECK-IADISABLED-NEXT:    zip1 v1.2d, v3.2d, v2.2d
+; CHECK-IADISABLED-NEXT:    zip2 v2.2d, v3.2d, v2.2d
+; CHECK-IADISABLED-NEXT:    zip1 v3.2d, v4.2d, v5.2d
+; CHECK-IADISABLED-NEXT:    zip2 v4.2d, v4.2d, v5.2d
+; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT:    ret
+  %v0 = load <4 x i64>, ptr %input0, align 8
+  %v1 = load <4 x i64>, ptr %input1, align 8
+  %v2 = load <4 x i64>, ptr %input2, align 8
+  %v3 = load <4 x i64>, ptr %input3, align 8
+  %interleave = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+  store <16 x i64> %interleave, ptr %output, align 8
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i32_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s1, w5
+; CHECK-NEXT:    fmov s3, w1
+; CHECK-NEXT:    ldr s0, [sp, #56]
+; CHECK-NEXT:    add x8, sp, #24
+; CHECK-NEXT:    ldr s2, [sp]
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    mov v0.s[1], w4
+; CHECK-NEXT:    ldr s4, [sp, #32]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    mov v3.s[1], w7
+; CHECK-NEXT:    ld1 { v2.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #72
+; CHECK-NEXT:    ld1 { v1.s }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #40
+; CHECK-NEXT:    ld1 { v0.s }[2], [x9]
+; CHECK-NEXT:    mov v2.s[2], w3
+; CHECK-NEXT:    ld1 { v3.s }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #64
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    mov v1.s[3], w6
+; CHECK-NEXT:    ld1 { v0.s }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    mov v3.s[3], w2
+; CHECK-NEXT:    ld1 { v4.s }[1], [x9]
+; CHECK-NEXT:    ld1 { v2.s }[3], [x8]
+; CHECK-NEXT:    str d4, [x0, #64]
+; CHECK-NEXT:    stp q3, q2, [x0]
+; CHECK-NEXT:    stp q0, q1, [x0, #32]
+; CHECK-NEXT:    ret
+  %interleave = call <18 x i32> @llvm.vector.interleave3.v18i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2)
+  store <18 x i32> %interleave, ptr %output, align 4
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i32_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s5, w6
+; CHECK-NEXT:    fmov s4, w5
+; CHECK-NEXT:    add x9, sp, #32
+; CHECK-NEXT:    fmov s1, w2
+; CHECK-NEXT:    fmov s3, w4
+; CHECK-NEXT:    mov x10, sp
+; CHECK-NEXT:    fmov s2, w3
+; CHECK-NEXT:    fmov s0, w1
+; CHECK-NEXT:    add x11, sp, #24
+; CHECK-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    ld1 { v4.s }[1], [x11]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #8
+; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
+; CHECK-NEXT:    mov v0.s[1], w7
+; CHECK-NEXT:    ld1 { v2.s }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #72
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    add x11, sp, #64
+; CHECK-NEXT:    ld1 { v4.s }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #56
+; CHECK-NEXT:    add x8, sp, #40
+; CHECK-NEXT:    ld1 { v5.s }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #48
+; CHECK-NEXT:    ld1 { v3.s }[2], [x11]
+; CHECK-NEXT:    ld1 { v2.s }[2], [x10]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x9]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x8]
+; CHECK-NEXT:    add x9, sp, #112
+; CHECK-NEXT:    add x8, sp, #104
+; CHECK-NEXT:    add x11, sp, #128
+; CHECK-NEXT:    add x10, sp, #120
+; CHECK-NEXT:    ld1 { v3.s }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #96
+; CHECK-NEXT:    ld1 { v2.s }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #88
+; CHECK-NEXT:    ld1 { v5.s }[3], [x11]
+; CHECK-NEXT:    ld1 { v4.s }[3], [x10]
+; CHECK-NEXT:    ld1 { v1.s }[3], [x9]
+; CHECK-NEXT:    ld1 { v0.s }[3], [x8]
+; CHECK-NEXT:    stp q2, q3, [x0, #32]
+; CHECK-NEXT:    str q5, [x0, #80]
+; CHECK-NEXT:    str q4, [x0, #64]
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %interleave = call <24 x i32> @llvm.vector.interleave4.v24i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3)
+  store <24 x i32> %interleave, ptr %output, align 4
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i16_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #192
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr d3, [x1, #16]
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    ldr q1, [x2]
+; CHECK-NEXT:    ldr d4, [x2, #16]
+; CHECK-NEXT:    ldr q2, [x3]
+; CHECK-NEXT:    ldr d5, [x3, #16]
+; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x8]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x8]
+; CHECK-NEXT:    ldr q0, [sp, #32]
+; CHECK-NEXT:    dup v1.2d, v0.d[1]
+; CHECK-NEXT:    str d1, [sp, #144]
+; CHECK-NEXT:    ldr q1, [sp, #64]
+; CHECK-NEXT:    ldr q3, [sp, #144]
+; CHECK-NEXT:    str d1, [sp, #112]
+; CHECK-NEXT:    ldp q2, q1, [sp]
+; CHECK-NEXT:    ext v0.16b, v1.16b, v0.16b, #8
+; CHECK-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NEXT:    ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-NEXT:    ldr q3, [sp, #48]
+; CHECK-NEXT:    stp q2, q1, [x0]
+; CHECK-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-NEXT:    ldr q0, [sp, #112]
+; CHECK-NEXT:    str d0, [x0, #64]
+; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    ret
+  %v0 = load <12 x i16>, ptr %input0, align 2
+  %v1 = load <12 x i16>, ptr %input1, align 2
+  %v2 = load <12 x i16>, ptr %input2, align 2
+  %interleave = call <36 x i16> @llvm.vector.interleave3.v36i16(<12 x i16> %v0, <12 x i16> %v1, <12 x i16> %v2)
+  store <36 x i16> %interleave, ptr %output, align 2
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i16_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x1, #16]
+; CHECK-NEXT:    ldr d2, [x2, #16]
+; CHECK-NEXT:    ldr d3, [x3, #16]
+; CHECK-NEXT:    ldr q4, [x4]
+; CHECK-NEXT:    ldr d5, [x4, #16]
+; CHECK-NEXT:    ldr q6, [x1]
+; CHECK-NEXT:    ldr q7, [x3]
+; CHECK-NEXT:    zip1 v2.8h, v2.8h, v5.8h
+; CHECK-NEXT:    zip1 v1.8h, v1.8h, v3.8h
+; CHECK-NEXT:    zip2 v3.8h, v0.8h, v4.8h
+; CHECK-NEXT:    zip2 v5.8h, v6.8h, v7.8h
+; CHECK-NEXT:    zip1 v0.8h, v0.8h, v4.8h
+; CHECK-NEXT:    zip1 v4.8h, v6.8h, v7.8h
+; CHECK-NEXT:    zip2 v6.8h, v1.8h, v2.8h
+; CHECK-NEXT:    zip1 v1.8h, v1.8h, v2.8h
+; CHECK-NEXT:    zip1 v7.8h, v5.8h, v3.8h
+; CHECK-NEXT:    zip2 v16.8h, v4.8h, v0.8h
+; CHECK-NEXT:    zip1 v0.8h, v4.8h, v0.8h
+; CHECK-NEXT:    zip2 v3.8h, v5.8h, v3.8h
+; CHECK-NEXT:    dup v2.2d, v6.d[1]
+; CHECK-NEXT:    dup v17.2d, v7.d[1]
+; CHECK-NEXT:    ext v7.16b, v16.16b, v7.16b, #8
+; CHECK-NEXT:    str d2, [sp, #16]
+; CHECK-NEXT:    ext v2.16b, v1.16b, v6.16b, #8
+; CHECK-NEXT:    str d17, [sp, #80]
+; CHECK-NEXT:    ldr q5, [sp, #16]
+; CHECK-NEXT:    mov v16.d[1], v7.d[0]
+; CHECK-NEXT:    ldr q4, [sp, #80]
+; CHECK-NEXT:    ext v4.16b, v7.16b, v4.16b, #8
+; CHECK-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-NEXT:    stp q0, q16, [x0]
+; CHECK-NEXT:    ext v0.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT:    stp q4, q3, [x0, #32]
+; CHECK-NEXT:    stp q1, q0, [x0, #64]
+; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    ret
+  %v0 = load <12 x i16>, ptr %input0, align 2
+  %v1 = load <12 x i16>, ptr %input1, align 2
+  %v2 = load <12 x i16>, ptr %input2, align 2
+  %v3 = load <12 x i16>, ptr %input3, align 2
+  %interleave = call <48 x i16> @llvm.vector.interleave4.v48i16(<12 x i16> %v0, <12 x i16> %v1, <12 x i16> %v2, <12 x i16> %v3)
+  store <48 x i16> %interleave, ptr %output, align 2
+  ret void
+}
+
+define void @wide_vector_interleave_st3_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i8_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #192
+; CHECK-NEXT:    .cfi_def_cfa_offset 192
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr d3, [x1, #16]
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    ldr q1, [x2]
+; CHECK-NEXT:    ldr d4, [x2, #16]
+; CHECK-NEXT:    ldr q2, [x3]
+; CHECK-NEXT:    ldr d5, [x3, #16]
+; CHECK-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x8]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x8]
+; CHECK-NEXT:    ldr q0, [sp, #32]
+; CHECK-NEXT:    dup v1.2d, v0.d[1]
+; CHECK-NEXT:    str d1, [sp, #144]
+; CHECK-NEXT:    ldr q1, [sp, #64]
+; CHECK-NEXT:    ldr q3, [sp, #144]
+; CHECK-NEXT:    str d1, [sp, #112]
+; CHECK-NEXT:    ldp q2, q1, [sp]
+; CHECK-NEXT:    ext v0.16b, v1.16b, v0.16b, #8
+; CHECK-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NEXT:    ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-NEXT:    ldr q3, [sp, #48]
+; CHECK-NEXT:    stp q2, q1, [x0]
+; CHECK-NEXT:    stp q0, q3, [x0, #32]
+; CHECK-NEXT:    ldr q0, [sp, #112]
+; CHECK-NEXT:    str d0, [x0, #64]
+; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    ret
+  %v0 = load <24 x i8>, ptr %input0, align 1
+  %v1 = load <24 x i8>, ptr %input1, align 1
+  %v2 = load <24 x i8>, ptr %input2, align 1
+  %interleave = call <72 x i8> @llvm.vector.interleave3.v72i8(<24 x i8> %v0, <24 x i8> %v1, <24 x i8> %v2)
+  store <72 x i8> %interleave, ptr %output, align 1
+  ret void
+}
+
+define void @wide_vector_interleave_st4_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i8_dreg:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #128
+; CHECK-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NEXT:    ldr q0, [x2]
+; CHECK-NEXT:    ldr d1, [x1, #16]
+; CHECK-NEXT:    ldr d2, [x2, #16]
+; CHECK-NEXT:    ldr d3, [x3, #16]
+; CHECK-NEXT:    ldr q4, [x4]
+; CHECK-NEXT:    ldr d5, [x4, #16]
+; CHECK-NEXT:    ldr q6, [x1]
+; CHECK-NEXT:    ldr q7, [x3]
+; CHECK-NEXT:    zip1 v2.16b, v2.16b, v5.16b
+; CHECK-NEXT:    zip1 v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    zip2 v3.16b, v0.16b, v4.16b
+; CHECK-NEXT:    zip2 v5.16b, v6.16b, v7.16b
+; CHECK-NEXT:    zip1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    zip1 v4.16b, v6.16b, v7.16b
+; CHECK-NEXT:    zip2 v6.16b, v1.16b, v2.16b
+; CHECK-NEXT:    zip1 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    zip1 v7.16b, v5.16b, v3.16b
+; CHECK-NEXT:    zip2 v16.16b, v4.16b, v0.16b
+; CHECK-NEXT:    zip1 v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    zip2 v3.16b, v5.16b, v3.16b
+; CHECK-NEXT:    dup v2.2d, v6.d[1]
+; CHECK-NEXT:    dup v17.2d, v7.d[1]
+; CHECK-NEXT:    ext v7.16b, v16.16b, v7.16b, #8
+; CHECK-NEXT:    str d2, [sp, #16]
+; CHECK-NEXT:    ext v2.16b, v1.16b, v6.16b, #8
+; CHECK-NEXT:    str d17, [sp, #80]
+; CHECK-NEXT:    ldr q5, [sp, #16]
+; CHECK-NEXT:    mov v16.d[1], v7.d[0]
+; CHECK-NEXT:    ldr q4, [sp, #80]
+; CHECK-NEXT:    ext v4.16b, v7.16b, v4.16b, #8
+; CHECK-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-NEXT:    stp q0, q16, [x0]
+; CHECK-NEXT:    ext v0.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT:    stp q4, q3, [x0, #32]
+; CHECK-NEXT:    stp q1, q0, [x0, #64]
+; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    ret
+  %v0 = load <24 x i8>, ptr %input0, align 1
+  %v1 = load <24 x i8>, ptr %input1, align 1
+  %v2 = load <24 x i8>, ptr %input2, align 1
+  %v3 = load <24 x i8>, ptr %input3, align 1
+  %interleave = call <96 x i8> @llvm.vector.interleave4.v96i8(<24 x i8> %v0, <24 x i8> %v1, <24 x i8> %v2, <24 x i8> %v3)
+  store <96 x i8> %interleave, ptr %output, align 1
+  ret void
+}

>From decbe8499b4e7c4dc29834e5ffe422f9cdd8aad0 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Fri, 21 Aug 2026 09:30:03 +0100
Subject: [PATCH 2/2] [AArch64] Support wide interleaved store combine

Combine will generate optimal stN sequence for wider types
instead of plain store and zip sequence
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  73 ++-
 .../AArch64/fixed-length-sve-interleave.ll    |  31 +-
 .../CodeGen/AArch64/interleaved-accesses.ll   |  95 +--
 .../AArch64/sve-interleaved-accesses.ll       |  93 +--
 .../AArch64/vector-interleave-store.ll        | 609 +++++-------------
 5 files changed, 285 insertions(+), 616 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 36a86241fe3b2..bdfdd4b0c410f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -28013,8 +28013,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 
   EVT SubVecTy = ValueInterleaveOps[0].getValueType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (!TLI.isTypeLegal(SubVecTy))
-    return SDValue();
   bool IsScalable = SubVecTy.isScalableVector();
   unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue();
   if (IsScalable && SubBits != 128)
@@ -28022,6 +28020,8 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 
   auto *MemN = cast<MemSDNode>(N);
   if (IsScalable) {
+    if (!TLI.isTypeLegal(SubVecTy))
+      return SDValue();
     SDValue Pred;
     if (IsMasked) {
       Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
@@ -28055,26 +28055,71 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 
   const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
   // Fixed length vector using NEON
-  if (!IsMasked && (SubBits == 64 || SubBits == 128) &&
-      Subtarget.isNeonAvailable()) {
+  if (!IsMasked && Subtarget.isNeonAvailable()) {
     static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
                                                    Intrinsic::aarch64_neon_st3,
                                                    Intrinsic::aarch64_neon_st4};
-    SmallVector<SDValue, 8> Ops;
-    Ops.push_back(Chain);
-    Ops.push_back(
-        DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64));
-    Ops.append(ValueInterleaveOps);
-    Ops.push_back(BasePtr);
-    return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
-                                   DAG.getVTList(MVT::Other), Ops,
-                                   MemN->getMemoryVT(), MemN->getMemOperand());
+    SDValue IID = DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64);
+    // Split fixed-length interleaves whose inputs are wider than a NEON
+    // register into a sequence of stN. Type legalization would
+    // otherwise split the interleave itself, leaving ZIPs and plain stores
+    // that can no longer be recognized as an interleaved store.
+
+    if (SubBits % 64 != 0)
+      return SDValue();
+
+    EVT EltVT = SubVecTy.getVectorElementType();
+    unsigned EltBits = EltVT.getSizeInBits();
+    auto IsLegalChunk = [&](unsigned ChunkBits) {
+      if (ChunkBits % EltBits != 0)
+        return false;
+      EVT ChunkVT =
+          EVT::getVectorVT(*DAG.getContext(), EltVT, ChunkBits / EltBits);
+      return TLI.isTypeLegal(ChunkVT);
+    };
+
+    unsigned Num128BitChunks = SubBits / 128;
+    bool Has64BitTail = SubBits % 128 != 0;
+    if ((Num128BitChunks && !IsLegalChunk(128)) ||
+        (Has64BitTail && !IsLegalChunk(64)))
+      return SDValue();
+
+    MachineFunction &MF = DAG.getMachineFunction();
+    MachineMemOperand *MMO = MemN->getMemOperand();
+    SmallVector<SDValue, 8> StoreChains;
+    unsigned NumChunks = Num128BitChunks + Has64BitTail;
+    StoreChains.reserve(NumChunks);
+    unsigned Offset = 0;
+    unsigned EltOffset = 0;
+    for (unsigned I = 0; I != NumChunks; ++I) {
+      unsigned ChunkBits = I < Num128BitChunks ? 128 : 64;
+      unsigned NarrowNumElts = ChunkBits / EltBits;
+      EVT NarrowVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NarrowNumElts);
+      unsigned ChunkBytes = NumParts * ChunkBits / 8;
+      EVT ChunkMemVT =
+          EVT::getVectorVT(*DAG.getContext(), EltVT, NarrowNumElts * NumParts);
+      SDValue Ptr =
+          DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(Offset));
+      SmallVector<SDValue, 8> Ops = {Chain, IID};
+      for (SDValue V : ValueInterleaveOps)
+        Ops.push_back(DAG.getExtractSubvector(DL, NarrowVT, V, EltOffset));
+      Ops.push_back(Ptr);
+      MachineMemOperand *ChunkMMO =
+          MF.getMachineMemOperand(MMO, Offset, ChunkBytes);
+      StoreChains.push_back(DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+                                                    DAG.getVTList(MVT::Other),
+                                                    Ops, ChunkMemVT, ChunkMMO));
+      Offset += ChunkBytes;
+      EltOffset += NarrowNumElts;
+    }
+    return DAG.getTokenFactor(DL, StoreChains);
   }
 
   // Fixed-length vectors using SVE
   if (!Subtarget.isSVEorStreamingSVEAvailable())
     return SDValue();
-
+  if (!TLI.isTypeLegal(SubVecTy))
+    return SDValue();
   EVT ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy);
   SDValue Pred;
   if (IsMasked) {
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
index 1278606f5b63f..4bdd5bfb4588d 100644
--- a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -29,17 +29,13 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
 ; CHECK-LABEL: store_factor3_intrinsic:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v18.16b, v4.16b
-; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v4.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT:    splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT:    mov v3.16b, v1.16b
+; CHECK-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
   store <24 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -49,21 +45,16 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
 define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) vscale_range(2,2) {
 ; CHECK-LABEL: store_factor4_intrinsic:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
 ; CHECK-NEXT:    mov v19.16b, v6.16b
-; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    mov v18.16b, v4.16b
+; CHECK-NEXT:    mov v6.16b, v5.16b
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v5.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT:    splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT:    splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT:    splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT:    mov v4.16b, v1.16b
+; CHECK-NEXT:    st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
   store <16 x i64> %interleaved.vec, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index bedce9ec6a86e..2e52b3b373484 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -2331,40 +2331,17 @@ define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 }
 
 define void @store_factor3_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-IAENABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IAENABLED:       // %bb.0:
-; NEON-IAENABLED-NEXT:    mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IAENABLED-NEXT:    mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT:    mov v4.16b, v3.16b
-; NEON-IAENABLED-NEXT:    mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT:    mov v3.16b, v1.16b
-; NEON-IAENABLED-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
-; NEON-IAENABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; NEON-IAENABLED-NEXT:    ret
-;
-; NEON-IADISABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IADISABLED:       // %bb.0:
-; NEON-IADISABLED-NEXT:    sub sp, sp, #96
-; NEON-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
-; NEON-IADISABLED-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IADISABLED-NEXT:    mov v18.16b, v4.16b
-; NEON-IADISABLED-NEXT:    mov x8, sp
-; NEON-IADISABLED-NEXT:    mov v17.16b, v2.16b
-; NEON-IADISABLED-NEXT:    mov v4.16b, v3.16b
-; NEON-IADISABLED-NEXT:    mov v16.16b, v0.16b
-; NEON-IADISABLED-NEXT:    mov v3.16b, v1.16b
-; NEON-IADISABLED-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x8]
-; NEON-IADISABLED-NEXT:    add x8, sp, #48
-; NEON-IADISABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x8]
-; NEON-IADISABLED-NEXT:    ldp q2, q0, [sp, #64]
-; NEON-IADISABLED-NEXT:    ldp q5, q1, [sp, #32]
-; NEON-IADISABLED-NEXT:    ldp q3, q4, [sp]
-; NEON-IADISABLED-NEXT:    stp q2, q0, [x0, #64]
-; NEON-IADISABLED-NEXT:    stp q5, q1, [x0, #32]
-; NEON-IADISABLED-NEXT:    stp q3, q4, [x0]
-; NEON-IADISABLED-NEXT:    add sp, sp, #96
-; NEON-IADISABLED-NEXT:    ret
+; NEON-LABEL: store_factor3_wide_intrinsic:
+; NEON:       // %bb.0:
+; NEON-NEXT:    mov v18.16b, v4.16b
+; NEON-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-NEXT:    mov v17.16b, v2.16b
+; NEON-NEXT:    mov v4.16b, v3.16b
+; NEON-NEXT:    mov v16.16b, v0.16b
+; NEON-NEXT:    mov v3.16b, v1.16b
+; NEON-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-NEXT:    ret
 ;
 ; NO_NEON-LABEL: store_factor3_wide_intrinsic:
 ; NO_NEON:       // %bb.0:
@@ -2458,43 +2435,19 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 }
 
 define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-IAENABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IAENABLED:       // %bb.0:
-; NEON-IAENABLED-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
-; NEON-IAENABLED-NEXT:    mov v19.16b, v6.16b
-; NEON-IAENABLED-NEXT:    mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT:    mov v6.16b, v5.16b
-; NEON-IAENABLED-NEXT:    mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT:    mov v5.16b, v3.16b
-; NEON-IAENABLED-NEXT:    mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT:    mov v4.16b, v1.16b
-; NEON-IAENABLED-NEXT:    st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
-; NEON-IAENABLED-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; NEON-IAENABLED-NEXT:    ret
-;
-; NEON-IADISABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IADISABLED:       // %bb.0:
-; NEON-IADISABLED-NEXT:    zip2 v16.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT:    zip2 v17.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT:    zip1 v3.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT:    zip1 v1.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT:    zip2 v5.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT:    zip2 v7.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT:    zip1 v2.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT:    zip1 v0.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT:    zip2 v18.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT:    zip1 v16.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT:    zip2 v4.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT:    zip1 v1.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT:    zip2 v3.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT:    zip1 v5.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT:    zip2 v6.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT:    zip1 v0.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT:    stp q16, q18, [x0, #96]
-; NEON-IADISABLED-NEXT:    stp q1, q4, [x0, #64]
-; NEON-IADISABLED-NEXT:    stp q0, q6, [x0]
-; NEON-IADISABLED-NEXT:    stp q5, q3, [x0, #32]
-; NEON-IADISABLED-NEXT:    ret
+; NEON-LABEL: store_factor4_wide_intrinsic:
+; NEON:       // %bb.0:
+; NEON-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-NEXT:    mov v19.16b, v6.16b
+; NEON-NEXT:    mov v18.16b, v4.16b
+; NEON-NEXT:    mov v6.16b, v5.16b
+; NEON-NEXT:    mov v17.16b, v2.16b
+; NEON-NEXT:    mov v5.16b, v3.16b
+; NEON-NEXT:    mov v16.16b, v0.16b
+; NEON-NEXT:    mov v4.16b, v1.16b
+; NEON-NEXT:    st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-NEXT:    ret
 ;
 ; NO_NEON-LABEL: store_factor4_wide_intrinsic:
 ; NO_NEON:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index 30f944e0a050d..faa49f7bb25b9 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1678,17 +1678,13 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
 ; CHECK-LABEL: store_factor3_intrinsic:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v18.16b, v4.16b
-; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v4.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT:    splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT:    mov v3.16b, v1.16b
+; CHECK-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
   store <24 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -1698,21 +1694,16 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
 define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
 ; CHECK-LABEL: store_factor4_intrinsic:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
 ; CHECK-NEXT:    mov v19.16b, v6.16b
-; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    mov v18.16b, v4.16b
+; CHECK-NEXT:    mov v6.16b, v5.16b
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v5.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT:    splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT:    splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT:    splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT:    mov v4.16b, v1.16b
+; CHECK-NEXT:    st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
   store <16 x i64> %interleaved.vec, ptr %ptr, align 4
@@ -1746,17 +1737,13 @@ define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %
 ; CHECK-LABEL: store_ptrvec_factor3_intrinsic:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v18.16b, v4.16b
-; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v4.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT:    splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT:    splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st3d { z16.d - z18.d }, p0, [x0]
+; CHECK-NEXT:    mov v3.16b, v1.16b
+; CHECK-NEXT:    st3 { v16.2d, v17.2d, v18.2d }, [x0], #48
+; CHECK-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <12 x ptr> @llvm.vector.interleave3.v12p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2)
   store <12 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -1766,21 +1753,16 @@ define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %
 define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
 ; CHECK-LABEL: store_ptrvec_factor4_intrinsic:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
 ; CHECK-NEXT:    mov v19.16b, v6.16b
-; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    mov v18.16b, v4.16b
+; CHECK-NEXT:    mov v6.16b, v5.16b
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v5.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT:    splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT:    splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT:    splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT:    mov v4.16b, v1.16b
+; CHECK-NEXT:    st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <16 x ptr> @llvm.vector.interleave4.v16p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3)
   store <16 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -1871,21 +1853,16 @@ define void @store_min_ge_type_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1)
 define void @store_double_factor4_intrinsic(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
 ; CHECK-LABEL: store_double_factor4_intrinsic:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
 ; CHECK-NEXT:    mov v19.16b, v6.16b
-; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    mov v18.16b, v4.16b
+; CHECK-NEXT:    mov v6.16b, v5.16b
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v5.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT:    splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT:    splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT:    splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT:    mov v4.16b, v1.16b
+; CHECK-NEXT:    st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <16 x double> @llvm.vector.interleave4.v16f64(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3)
   store <16 x double> %interleaved.vec, ptr %ptr, align 4
@@ -1896,17 +1873,13 @@ define void @store_float_factor3_intrinsic(ptr %ptr, <8 x float> %v0, <8 x float
 ; CHECK-LABEL: store_float_factor3_intrinsic:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v18.16b, v4.16b
-; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
 ; CHECK-NEXT:    mov v17.16b, v2.16b
+; CHECK-NEXT:    mov v4.16b, v3.16b
 ; CHECK-NEXT:    mov v16.16b, v0.16b
-; CHECK-NEXT:    splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT:    splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT:    splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT:    mov v3.16b, v1.16b
+; CHECK-NEXT:    st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
 ; CHECK-NEXT:    ret
   %interleaved.vec = call <24 x float> @llvm.vector.interleave3.v24f32(<8 x float> %v0, <8 x float> %v1, <8 x float> %v2)
   store <24 x float> %interleaved.vec, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index 9732dfc246e70..ce07e968968b3 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -96,34 +96,14 @@ define void @wide_vector_interleave_st2(ptr %input0, ptr %input1, ptr %output) {
 }
 
 define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-; CHECK-IAENABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
-; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT:    mov x8, sp
-; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x8]
-; CHECK-IADISABLED-NEXT:    add x8, sp, #48
-; CHECK-IADISABLED-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x8]
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT:    add sp, sp, #96
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q4, [x2]
+; CHECK-NEXT:    ldp q2, q5, [x3]
+; CHECK-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-NEXT:    st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <8 x i32>, ptr %input0, align 4
   %v1 = load <8 x i32>, ptr %input1, align 4
   %v2 = load <8 x i32>, ptr %input2, align 4
@@ -133,43 +113,15 @@ define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, p
 }
 
 define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT:    st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IAENABLED-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT:    zip2 v16.4s, v0.4s, v2.4s
-; CHECK-IADISABLED-NEXT:    zip1 v0.4s, v0.4s, v2.4s
-; CHECK-IADISABLED-NEXT:    zip2 v2.4s, v5.4s, v7.4s
-; CHECK-IADISABLED-NEXT:    zip2 v17.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT:    zip1 v1.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT:    zip2 v3.4s, v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT:    zip1 v5.4s, v5.4s, v7.4s
-; CHECK-IADISABLED-NEXT:    zip1 v4.4s, v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT:    zip1 v18.4s, v17.4s, v16.4s
-; CHECK-IADISABLED-NEXT:    zip2 v16.4s, v17.4s, v16.4s
-; CHECK-IADISABLED-NEXT:    zip1 v6.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT:    zip2 v0.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT:    zip1 v1.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT:    zip2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT:    zip1 v3.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT:    zip2 v4.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q4, [x1]
+; CHECK-NEXT:    ldp q1, q5, [x2]
+; CHECK-NEXT:    ldp q2, q6, [x3]
+; CHECK-NEXT:    ldp q3, q7, [x4]
+; CHECK-NEXT:    st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <8 x i32>, ptr %input0, align 4
   %v1 = load <8 x i32>, ptr %input1, align 4
   %v2 = load <8 x i32>, ptr %input2, align 4
@@ -180,34 +132,14 @@ define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, p
 }
 
 define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i8:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
-; CHECK-IAENABLED-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i8:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
-; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT:    mov x8, sp
-; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x8]
-; CHECK-IADISABLED-NEXT:    add x8, sp, #48
-; CHECK-IADISABLED-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x8]
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT:    add sp, sp, #96
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st3_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q4, [x2]
+; CHECK-NEXT:    ldp q2, q5, [x3]
+; CHECK-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
+; CHECK-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <32 x i8>, ptr %input0, align 1
   %v1 = load <32 x i8>, ptr %input1, align 1
   %v2 = load <32 x i8>, ptr %input2, align 1
@@ -217,43 +149,15 @@ define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1
 }
 
 define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i8:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT:    st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
-; CHECK-IAENABLED-NEXT:    st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i8:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT:    zip2 v16.16b, v0.16b, v2.16b
-; CHECK-IADISABLED-NEXT:    zip1 v0.16b, v0.16b, v2.16b
-; CHECK-IADISABLED-NEXT:    zip2 v2.16b, v5.16b, v7.16b
-; CHECK-IADISABLED-NEXT:    zip2 v17.16b, v3.16b, v1.16b
-; CHECK-IADISABLED-NEXT:    zip1 v1.16b, v3.16b, v1.16b
-; CHECK-IADISABLED-NEXT:    zip2 v3.16b, v4.16b, v6.16b
-; CHECK-IADISABLED-NEXT:    zip1 v5.16b, v5.16b, v7.16b
-; CHECK-IADISABLED-NEXT:    zip1 v4.16b, v4.16b, v6.16b
-; CHECK-IADISABLED-NEXT:    zip1 v18.16b, v17.16b, v16.16b
-; CHECK-IADISABLED-NEXT:    zip2 v16.16b, v17.16b, v16.16b
-; CHECK-IADISABLED-NEXT:    zip1 v6.16b, v1.16b, v0.16b
-; CHECK-IADISABLED-NEXT:    zip2 v0.16b, v1.16b, v0.16b
-; CHECK-IADISABLED-NEXT:    zip1 v1.16b, v3.16b, v2.16b
-; CHECK-IADISABLED-NEXT:    zip2 v2.16b, v3.16b, v2.16b
-; CHECK-IADISABLED-NEXT:    zip1 v3.16b, v4.16b, v5.16b
-; CHECK-IADISABLED-NEXT:    zip2 v4.16b, v4.16b, v5.16b
-; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st4_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q4, [x1]
+; CHECK-NEXT:    ldp q1, q5, [x2]
+; CHECK-NEXT:    ldp q2, q6, [x3]
+; CHECK-NEXT:    ldp q3, q7, [x4]
+; CHECK-NEXT:    st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+; CHECK-NEXT:    st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <32 x i8>, ptr %input0, align 1
   %v1 = load <32 x i8>, ptr %input1, align 1
   %v2 = load <32 x i8>, ptr %input2, align 1
@@ -264,34 +168,14 @@ define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1
 }
 
 define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i16:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
-; CHECK-IAENABLED-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i16:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
-; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT:    mov x8, sp
-; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x8]
-; CHECK-IADISABLED-NEXT:    add x8, sp, #48
-; CHECK-IADISABLED-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x8]
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT:    add sp, sp, #96
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st3_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q4, [x2]
+; CHECK-NEXT:    ldp q2, q5, [x3]
+; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
+; CHECK-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <16 x i16>, ptr %input0, align 2
   %v1 = load <16 x i16>, ptr %input1, align 2
   %v2 = load <16 x i16>, ptr %input2, align 2
@@ -301,43 +185,15 @@ define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input
 }
 
 define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i16:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT:    st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
-; CHECK-IAENABLED-NEXT:    st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i16:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT:    zip2 v16.8h, v0.8h, v2.8h
-; CHECK-IADISABLED-NEXT:    zip1 v0.8h, v0.8h, v2.8h
-; CHECK-IADISABLED-NEXT:    zip2 v2.8h, v5.8h, v7.8h
-; CHECK-IADISABLED-NEXT:    zip2 v17.8h, v3.8h, v1.8h
-; CHECK-IADISABLED-NEXT:    zip1 v1.8h, v3.8h, v1.8h
-; CHECK-IADISABLED-NEXT:    zip2 v3.8h, v4.8h, v6.8h
-; CHECK-IADISABLED-NEXT:    zip1 v5.8h, v5.8h, v7.8h
-; CHECK-IADISABLED-NEXT:    zip1 v4.8h, v4.8h, v6.8h
-; CHECK-IADISABLED-NEXT:    zip1 v18.8h, v17.8h, v16.8h
-; CHECK-IADISABLED-NEXT:    zip2 v16.8h, v17.8h, v16.8h
-; CHECK-IADISABLED-NEXT:    zip1 v6.8h, v1.8h, v0.8h
-; CHECK-IADISABLED-NEXT:    zip2 v0.8h, v1.8h, v0.8h
-; CHECK-IADISABLED-NEXT:    zip1 v1.8h, v3.8h, v2.8h
-; CHECK-IADISABLED-NEXT:    zip2 v2.8h, v3.8h, v2.8h
-; CHECK-IADISABLED-NEXT:    zip1 v3.8h, v4.8h, v5.8h
-; CHECK-IADISABLED-NEXT:    zip2 v4.8h, v4.8h, v5.8h
-; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st4_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q4, [x1]
+; CHECK-NEXT:    ldp q1, q5, [x2]
+; CHECK-NEXT:    ldp q2, q6, [x3]
+; CHECK-NEXT:    ldp q3, q7, [x4]
+; CHECK-NEXT:    st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
+; CHECK-NEXT:    st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <16 x i16>, ptr %input0, align 2
   %v1 = load <16 x i16>, ptr %input1, align 2
   %v2 = load <16 x i16>, ptr %input2, align 2
@@ -348,34 +204,14 @@ define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input
 }
 
 define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i64:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
-; CHECK-IAENABLED-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i64:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    sub sp, sp, #96
-; CHECK-IADISABLED-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT:    mov x8, sp
-; CHECK-IADISABLED-NEXT:    ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x8]
-; CHECK-IADISABLED-NEXT:    add x8, sp, #48
-; CHECK-IADISABLED-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x8]
-; CHECK-IADISABLED-NEXT:    ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT:    ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT:    ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT:    stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT:    add sp, sp, #96
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st3_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q1, q4, [x2]
+; CHECK-NEXT:    ldp q2, q5, [x3]
+; CHECK-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; CHECK-NEXT:    st3 { v3.2d, v4.2d, v5.2d }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <4 x i64>, ptr %input0, align 8
   %v1 = load <4 x i64>, ptr %input1, align 8
   %v2 = load <4 x i64>, ptr %input2, align 8
@@ -385,43 +221,15 @@ define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input
 }
 
 define void @wide_vector_interleave_st4_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i64:
-; CHECK-IAENABLED:       // %bb.0:
-; CHECK-IAENABLED-NEXT:    ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT:    ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT:    ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT:    ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT:    st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
-; CHECK-IAENABLED-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
-; CHECK-IAENABLED-NEXT:    ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i64:
-; CHECK-IADISABLED:       // %bb.0:
-; CHECK-IADISABLED-NEXT:    ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT:    ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT:    ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT:    ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT:    zip2 v16.2d, v0.2d, v2.2d
-; CHECK-IADISABLED-NEXT:    zip1 v0.2d, v0.2d, v2.2d
-; CHECK-IADISABLED-NEXT:    zip2 v2.2d, v5.2d, v7.2d
-; CHECK-IADISABLED-NEXT:    zip2 v17.2d, v3.2d, v1.2d
-; CHECK-IADISABLED-NEXT:    zip1 v1.2d, v3.2d, v1.2d
-; CHECK-IADISABLED-NEXT:    zip2 v3.2d, v4.2d, v6.2d
-; CHECK-IADISABLED-NEXT:    zip1 v5.2d, v5.2d, v7.2d
-; CHECK-IADISABLED-NEXT:    zip1 v4.2d, v4.2d, v6.2d
-; CHECK-IADISABLED-NEXT:    zip1 v18.2d, v17.2d, v16.2d
-; CHECK-IADISABLED-NEXT:    zip2 v16.2d, v17.2d, v16.2d
-; CHECK-IADISABLED-NEXT:    zip1 v6.2d, v1.2d, v0.2d
-; CHECK-IADISABLED-NEXT:    zip2 v0.2d, v1.2d, v0.2d
-; CHECK-IADISABLED-NEXT:    zip1 v1.2d, v3.2d, v2.2d
-; CHECK-IADISABLED-NEXT:    zip2 v2.2d, v3.2d, v2.2d
-; CHECK-IADISABLED-NEXT:    zip1 v3.2d, v4.2d, v5.2d
-; CHECK-IADISABLED-NEXT:    zip2 v4.2d, v4.2d, v5.2d
-; CHECK-IADISABLED-NEXT:    stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT:    stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT:    stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT:    stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT:    ret
+; CHECK-LABEL: wide_vector_interleave_st4_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q4, [x1]
+; CHECK-NEXT:    ldp q1, q5, [x2]
+; CHECK-NEXT:    ldp q2, q6, [x3]
+; CHECK-NEXT:    ldp q3, q7, [x4]
+; CHECK-NEXT:    st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
+; CHECK-NEXT:    st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
+; CHECK-NEXT:    ret
   %v0 = load <4 x i64>, ptr %input0, align 8
   %v1 = load <4 x i64>, ptr %input1, align 8
   %v2 = load <4 x i64>, ptr %input2, align 8
@@ -434,35 +242,35 @@ define void @wide_vector_interleave_st4_i64(ptr %output, ptr %input0, ptr %input
 define void @wide_vector_interleave_st3_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2) {
 ; CHECK-LABEL: wide_vector_interleave_st3_i32_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov s1, w5
-; CHECK-NEXT:    fmov s3, w1
-; CHECK-NEXT:    ldr s0, [sp, #56]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    ldr s2, [sp]
-; CHECK-NEXT:    add x9, sp, #16
-; CHECK-NEXT:    mov v0.s[1], w4
-; CHECK-NEXT:    ldr s4, [sp, #32]
-; CHECK-NEXT:    ld1 { v1.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #56
+; CHECK-NEXT:    ldr s0, [sp, #40]
+; CHECK-NEXT:    add x9, sp, #64
+; CHECK-NEXT:    ldr s1, [x8]
 ; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    mov v3.s[1], w7
-; CHECK-NEXT:    ld1 { v2.s }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    ld1 { v1.s }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v0.s }[2], [x9]
-; CHECK-NEXT:    mov v2.s[2], w3
-; CHECK-NEXT:    ld1 { v3.s }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #64
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    mov v1.s[3], w6
-; CHECK-NEXT:    ld1 { v0.s }[3], [x8]
+; CHECK-NEXT:    fmov s2, w1
+; CHECK-NEXT:    ld1 { v0.s }[1], [x8]
+; CHECK-NEXT:    fmov s3, w7
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    ld1 { v1.s }[1], [x9]
+; CHECK-NEXT:    fmov s5, w5
+; CHECK-NEXT:    ldr s7, [sp, #72]
+; CHECK-NEXT:    zip1 v4.2d, v0.2d, v1.2d
+; CHECK-NEXT:    ld1 { v3.s }[1], [x8]
 ; CHECK-NEXT:    add x8, sp, #8
-; CHECK-NEXT:    mov v3.s[3], w2
-; CHECK-NEXT:    ld1 { v4.s }[1], [x9]
-; CHECK-NEXT:    ld1 { v2.s }[3], [x8]
-; CHECK-NEXT:    str d4, [x0, #64]
-; CHECK-NEXT:    stp q3, q2, [x0]
-; CHECK-NEXT:    stp q0, q1, [x0, #32]
+; CHECK-NEXT:    ld1 { v3.s }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    ld1 { v3.s }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #80
+; CHECK-NEXT:    mov v2.s[1], w2
+; CHECK-NEXT:    ld1 { v7.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    ldr s6, [sp, #24]
+; CHECK-NEXT:    ld1 { v6.s }[1], [x8]
+; CHECK-NEXT:    mov v2.s[2], w3
+; CHECK-NEXT:    mov v5.s[1], w6
+; CHECK-NEXT:    mov v2.s[3], w4
+; CHECK-NEXT:    st3 { v2.4s, v3.4s, v4.4s }, [x0], #48
+; CHECK-NEXT:    st3 { v5.2s, v6.2s, v7.2s }, [x0]
 ; CHECK-NEXT:    ret
   %interleave = call <18 x i32> @llvm.vector.interleave3.v18i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2)
   store <18 x i32> %interleave, ptr %output, align 4
@@ -472,51 +280,46 @@ define void @wide_vector_interleave_st3_i32_dreg(ptr %output, <6 x i32> %v0, <6
 define void @wide_vector_interleave_st4_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3) {
 ; CHECK-LABEL: wide_vector_interleave_st4_i32_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov s5, w6
-; CHECK-NEXT:    fmov s4, w5
-; CHECK-NEXT:    add x9, sp, #32
-; CHECK-NEXT:    fmov s1, w2
-; CHECK-NEXT:    fmov s3, w4
-; CHECK-NEXT:    mov x10, sp
-; CHECK-NEXT:    fmov s2, w3
-; CHECK-NEXT:    fmov s0, w1
-; CHECK-NEXT:    add x11, sp, #24
-; CHECK-NEXT:    ld1 { v5.s }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #16
-; CHECK-NEXT:    ld1 { v4.s }[1], [x11]
-; CHECK-NEXT:    ld1 { v1.s }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #8
-; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
-; CHECK-NEXT:    mov v0.s[1], w7
-; CHECK-NEXT:    ld1 { v2.s }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #72
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    add x11, sp, #64
-; CHECK-NEXT:    ld1 { v4.s }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #56
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v5.s }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #48
-; CHECK-NEXT:    ld1 { v3.s }[2], [x11]
-; CHECK-NEXT:    ld1 { v2.s }[2], [x10]
-; CHECK-NEXT:    ld1 { v1.s }[2], [x9]
-; CHECK-NEXT:    ld1 { v0.s }[2], [x8]
+; CHECK-NEXT:    ldr s0, [sp, #88]
+; CHECK-NEXT:    add x8, sp, #96
+; CHECK-NEXT:    add x9, sp, #104
+; CHECK-NEXT:    ldr s1, [x9]
 ; CHECK-NEXT:    add x9, sp, #112
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    add x11, sp, #128
-; CHECK-NEXT:    add x10, sp, #120
-; CHECK-NEXT:    ld1 { v3.s }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #96
-; CHECK-NEXT:    ld1 { v2.s }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ld1 { v5.s }[3], [x11]
-; CHECK-NEXT:    ld1 { v4.s }[3], [x10]
-; CHECK-NEXT:    ld1 { v1.s }[3], [x9]
-; CHECK-NEXT:    ld1 { v0.s }[3], [x8]
-; CHECK-NEXT:    stp q2, q3, [x0, #32]
-; CHECK-NEXT:    str q5, [x0, #80]
-; CHECK-NEXT:    str q4, [x0, #64]
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ldr s2, [sp, #40]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #56
+; CHECK-NEXT:    fmov s4, w1
+; CHECK-NEXT:    ldr s3, [x8]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x9]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    add x9, sp, #64
+; CHECK-NEXT:    ld1 { v2.s }[1], [x8]
+; CHECK-NEXT:    fmov s5, w7
+; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    zip1 v7.2d, v0.2d, v1.2d
+; CHECK-NEXT:    zip1 v6.2d, v2.2d, v3.2d
+; CHECK-NEXT:    fmov s0, w5
+; CHECK-NEXT:    ld1 { v5.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    ldr s3, [sp, #120]
+; CHECK-NEXT:    ld1 { v5.s }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    ld1 { v5.s }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #128
+; CHECK-NEXT:    mov v4.s[1], w2
+; CHECK-NEXT:    ld1 { v3.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #80
+; CHECK-NEXT:    ldr s2, [sp, #72]
+; CHECK-NEXT:    ld1 { v2.s }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    mov v4.s[2], w3
+; CHECK-NEXT:    ldr s1, [sp, #24]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x8]
+; CHECK-NEXT:    mov v0.s[1], w6
+; CHECK-NEXT:    mov v4.s[3], w4
+; CHECK-NEXT:    st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0], #64
+; CHECK-NEXT:    st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
 ; CHECK-NEXT:    ret
   %interleave = call <24 x i32> @llvm.vector.interleave4.v24i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3)
   store <24 x i32> %interleave, ptr %output, align 4
@@ -526,34 +329,14 @@ define void @wide_vector_interleave_st4_i32_dreg(ptr %output, <6 x i32> %v0, <6
 define void @wide_vector_interleave_st3_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
 ; CHECK-LABEL: wide_vector_interleave_st3_i16_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #192
-; CHECK-NEXT:    .cfi_def_cfa_offset 192
 ; CHECK-NEXT:    ldr q0, [x1]
 ; CHECK-NEXT:    ldr d3, [x1, #16]
-; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ldr q1, [x2]
 ; CHECK-NEXT:    ldr d4, [x2, #16]
 ; CHECK-NEXT:    ldr q2, [x3]
 ; CHECK-NEXT:    ldr d5, [x3, #16]
-; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    st3 { v3.8h, v4.8h, v5.8h }, [x8]
-; CHECK-NEXT:    ldr q0, [sp, #32]
-; CHECK-NEXT:    dup v1.2d, v0.d[1]
-; CHECK-NEXT:    str d1, [sp, #144]
-; CHECK-NEXT:    ldr q1, [sp, #64]
-; CHECK-NEXT:    ldr q3, [sp, #144]
-; CHECK-NEXT:    str d1, [sp, #112]
-; CHECK-NEXT:    ldp q2, q1, [sp]
-; CHECK-NEXT:    ext v0.16b, v1.16b, v0.16b, #8
-; CHECK-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-NEXT:    ext v0.16b, v0.16b, v3.16b, #8
-; CHECK-NEXT:    ldr q3, [sp, #48]
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-NEXT:    ldr q0, [sp, #112]
-; CHECK-NEXT:    str d0, [x0, #64]
-; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
+; CHECK-NEXT:    st3 { v3.4h, v4.4h, v5.4h }, [x0]
 ; CHECK-NEXT:    ret
   %v0 = load <12 x i16>, ptr %input0, align 2
   %v1 = load <12 x i16>, ptr %input1, align 2
@@ -566,44 +349,16 @@ define void @wide_vector_interleave_st3_i16_dreg(ptr %output, ptr %input0, ptr %
 define void @wide_vector_interleave_st4_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
 ; CHECK-LABEL: wide_vector_interleave_st4_i16_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #128
-; CHECK-NEXT:    .cfi_def_cfa_offset 128
-; CHECK-NEXT:    ldr q0, [x2]
-; CHECK-NEXT:    ldr d1, [x1, #16]
-; CHECK-NEXT:    ldr d2, [x2, #16]
-; CHECK-NEXT:    ldr d3, [x3, #16]
-; CHECK-NEXT:    ldr q4, [x4]
-; CHECK-NEXT:    ldr d5, [x4, #16]
-; CHECK-NEXT:    ldr q6, [x1]
-; CHECK-NEXT:    ldr q7, [x3]
-; CHECK-NEXT:    zip1 v2.8h, v2.8h, v5.8h
-; CHECK-NEXT:    zip1 v1.8h, v1.8h, v3.8h
-; CHECK-NEXT:    zip2 v3.8h, v0.8h, v4.8h
-; CHECK-NEXT:    zip2 v5.8h, v6.8h, v7.8h
-; CHECK-NEXT:    zip1 v0.8h, v0.8h, v4.8h
-; CHECK-NEXT:    zip1 v4.8h, v6.8h, v7.8h
-; CHECK-NEXT:    zip2 v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    zip1 v1.8h, v1.8h, v2.8h
-; CHECK-NEXT:    zip1 v7.8h, v5.8h, v3.8h
-; CHECK-NEXT:    zip2 v16.8h, v4.8h, v0.8h
-; CHECK-NEXT:    zip1 v0.8h, v4.8h, v0.8h
-; CHECK-NEXT:    zip2 v3.8h, v5.8h, v3.8h
-; CHECK-NEXT:    dup v2.2d, v6.d[1]
-; CHECK-NEXT:    dup v17.2d, v7.d[1]
-; CHECK-NEXT:    ext v7.16b, v16.16b, v7.16b, #8
-; CHECK-NEXT:    str d2, [sp, #16]
-; CHECK-NEXT:    ext v2.16b, v1.16b, v6.16b, #8
-; CHECK-NEXT:    str d17, [sp, #80]
-; CHECK-NEXT:    ldr q5, [sp, #16]
-; CHECK-NEXT:    mov v16.d[1], v7.d[0]
-; CHECK-NEXT:    ldr q4, [sp, #80]
-; CHECK-NEXT:    ext v4.16b, v7.16b, v4.16b, #8
-; CHECK-NEXT:    mov v1.d[1], v2.d[0]
-; CHECK-NEXT:    stp q0, q16, [x0]
-; CHECK-NEXT:    ext v0.16b, v2.16b, v5.16b, #8
-; CHECK-NEXT:    stp q4, q3, [x0, #32]
-; CHECK-NEXT:    stp q1, q0, [x0, #64]
-; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr d4, [x1, #16]
+; CHECK-NEXT:    ldr q1, [x2]
+; CHECK-NEXT:    ldr d5, [x2, #16]
+; CHECK-NEXT:    ldr q2, [x3]
+; CHECK-NEXT:    ldr d6, [x3, #16]
+; CHECK-NEXT:    ldr q3, [x4]
+; CHECK-NEXT:    ldr d7, [x4, #16]
+; CHECK-NEXT:    st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
+; CHECK-NEXT:    st4 { v4.4h, v5.4h, v6.4h, v7.4h }, [x0]
 ; CHECK-NEXT:    ret
   %v0 = load <12 x i16>, ptr %input0, align 2
   %v1 = load <12 x i16>, ptr %input1, align 2
@@ -617,34 +372,14 @@ define void @wide_vector_interleave_st4_i16_dreg(ptr %output, ptr %input0, ptr %
 define void @wide_vector_interleave_st3_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
 ; CHECK-LABEL: wide_vector_interleave_st3_i8_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #192
-; CHECK-NEXT:    .cfi_def_cfa_offset 192
 ; CHECK-NEXT:    ldr q0, [x1]
 ; CHECK-NEXT:    ldr d3, [x1, #16]
-; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ldr q1, [x2]
 ; CHECK-NEXT:    ldr d4, [x2, #16]
 ; CHECK-NEXT:    ldr q2, [x3]
 ; CHECK-NEXT:    ldr d5, [x3, #16]
-; CHECK-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    st3 { v3.16b, v4.16b, v5.16b }, [x8]
-; CHECK-NEXT:    ldr q0, [sp, #32]
-; CHECK-NEXT:    dup v1.2d, v0.d[1]
-; CHECK-NEXT:    str d1, [sp, #144]
-; CHECK-NEXT:    ldr q1, [sp, #64]
-; CHECK-NEXT:    ldr q3, [sp, #144]
-; CHECK-NEXT:    str d1, [sp, #112]
-; CHECK-NEXT:    ldp q2, q1, [sp]
-; CHECK-NEXT:    ext v0.16b, v1.16b, v0.16b, #8
-; CHECK-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-NEXT:    ext v0.16b, v0.16b, v3.16b, #8
-; CHECK-NEXT:    ldr q3, [sp, #48]
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q0, q3, [x0, #32]
-; CHECK-NEXT:    ldr q0, [sp, #112]
-; CHECK-NEXT:    str d0, [x0, #64]
-; CHECK-NEXT:    add sp, sp, #192
+; CHECK-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
+; CHECK-NEXT:    st3 { v3.8b, v4.8b, v5.8b }, [x0]
 ; CHECK-NEXT:    ret
   %v0 = load <24 x i8>, ptr %input0, align 1
   %v1 = load <24 x i8>, ptr %input1, align 1
@@ -657,44 +392,16 @@ define void @wide_vector_interleave_st3_i8_dreg(ptr %output, ptr %input0, ptr %i
 define void @wide_vector_interleave_st4_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
 ; CHECK-LABEL: wide_vector_interleave_st4_i8_dreg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #128
-; CHECK-NEXT:    .cfi_def_cfa_offset 128
-; CHECK-NEXT:    ldr q0, [x2]
-; CHECK-NEXT:    ldr d1, [x1, #16]
-; CHECK-NEXT:    ldr d2, [x2, #16]
-; CHECK-NEXT:    ldr d3, [x3, #16]
-; CHECK-NEXT:    ldr q4, [x4]
-; CHECK-NEXT:    ldr d5, [x4, #16]
-; CHECK-NEXT:    ldr q6, [x1]
-; CHECK-NEXT:    ldr q7, [x3]
-; CHECK-NEXT:    zip1 v2.16b, v2.16b, v5.16b
-; CHECK-NEXT:    zip1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT:    zip2 v3.16b, v0.16b, v4.16b
-; CHECK-NEXT:    zip2 v5.16b, v6.16b, v7.16b
-; CHECK-NEXT:    zip1 v0.16b, v0.16b, v4.16b
-; CHECK-NEXT:    zip1 v4.16b, v6.16b, v7.16b
-; CHECK-NEXT:    zip2 v6.16b, v1.16b, v2.16b
-; CHECK-NEXT:    zip1 v1.16b, v1.16b, v2.16b
-; CHECK-NEXT:    zip1 v7.16b, v5.16b, v3.16b
-; CHECK-NEXT:    zip2 v16.16b, v4.16b, v0.16b
-; CHECK-NEXT:    zip1 v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    zip2 v3.16b, v5.16b, v3.16b
-; CHECK-NEXT:    dup v2.2d, v6.d[1]
-; CHECK-NEXT:    dup v17.2d, v7.d[1]
-; CHECK-NEXT:    ext v7.16b, v16.16b, v7.16b, #8
-; CHECK-NEXT:    str d2, [sp, #16]
-; CHECK-NEXT:    ext v2.16b, v1.16b, v6.16b, #8
-; CHECK-NEXT:    str d17, [sp, #80]
-; CHECK-NEXT:    ldr q5, [sp, #16]
-; CHECK-NEXT:    mov v16.d[1], v7.d[0]
-; CHECK-NEXT:    ldr q4, [sp, #80]
-; CHECK-NEXT:    ext v4.16b, v7.16b, v4.16b, #8
-; CHECK-NEXT:    mov v1.d[1], v2.d[0]
-; CHECK-NEXT:    stp q0, q16, [x0]
-; CHECK-NEXT:    ext v0.16b, v2.16b, v5.16b, #8
-; CHECK-NEXT:    stp q4, q3, [x0, #32]
-; CHECK-NEXT:    stp q1, q0, [x0, #64]
-; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    ldr d4, [x1, #16]
+; CHECK-NEXT:    ldr q1, [x2]
+; CHECK-NEXT:    ldr d5, [x2, #16]
+; CHECK-NEXT:    ldr q2, [x3]
+; CHECK-NEXT:    ldr d6, [x3, #16]
+; CHECK-NEXT:    ldr q3, [x4]
+; CHECK-NEXT:    ldr d7, [x4, #16]
+; CHECK-NEXT:    st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+; CHECK-NEXT:    st4 { v4.8b, v5.8b, v6.8b, v7.8b }, [x0]
 ; CHECK-NEXT:    ret
   %v0 = load <24 x i8>, ptr %input0, align 1
   %v1 = load <24 x i8>, ptr %input1, align 1



More information about the llvm-commits mailing list