[llvm] [AArch64] Support wide interleaved store combine (PR #217856)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 09:50:05 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/217856
>From 2d98230b614bd816078a2085bf143d29159ee909 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 20 Aug 2026 19:36:41 +0000
Subject: [PATCH 1/2] added the test
---
.../AArch64/vector-interleave-store.ll | 638 ++++++++++++++++++
1 file changed, 638 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index d3985991af531..9732dfc246e70 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+
define void @aarch64_vector_interleave_idx_st2(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1) {
; CHECK-IAENABLED-LABEL: aarch64_vector_interleave_idx_st2:
; CHECK-IAENABLED: // %bb.0: // %entry
@@ -66,3 +67,640 @@ entry:
store <16 x float> %interleave, ptr %gep1, align 16
ret void
}
+
+define void @wide_vector_interleave_st2(ptr %input0, ptr %input1, ptr %output) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q2, [x0]
+; CHECK-IAENABLED-NEXT: ldp q1, q3, [x1]
+; CHECK-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x2], #32
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x2]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q3, q0, [x1]
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [x0]
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: stp q4, q0, [x2, #32]
+; CHECK-IADISABLED-NEXT: stp q1, q2, [x2]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <8 x i32>, ptr %input0, align 4
+ %v1 = load <8 x i32>, ptr %input1, align 4
+ %interleaved.vec = tail call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
+ store <16 x i32> %interleaved.vec, ptr %output, align 8
+ ret void
+}
+
+define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #96
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #48
+; CHECK-IADISABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x8]
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT: add sp, sp, #96
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <8 x i32>, ptr %input0, align 4
+ %v1 = load <8 x i32>, ptr %input1, align 4
+ %v2 = load <8 x i32>, ptr %input2, align 4
+ %interleave = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleave, ptr %output, align 4
+ ret void
+}
+
+define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT: zip2 v16.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v5.4s, v7.4s
+; CHECK-IADISABLED-NEXT: zip2 v17.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip1 v5.4s, v5.4s, v7.4s
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip1 v18.4s, v17.4s, v16.4s
+; CHECK-IADISABLED-NEXT: zip2 v16.4s, v17.4s, v16.4s
+; CHECK-IADISABLED-NEXT: zip1 v6.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v3.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <8 x i32>, ptr %input0, align 4
+ %v1 = load <8 x i32>, ptr %input1, align 4
+ %v2 = load <8 x i32>, ptr %input2, align 4
+ %v3 = load <8 x i32>, ptr %input3, align 4
+ %interleave = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3)
+ store <32 x i32> %interleave, ptr %output, align 4
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i8:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
+; CHECK-IAENABLED-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i8:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #96
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #48
+; CHECK-IADISABLED-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x8]
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT: add sp, sp, #96
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <32 x i8>, ptr %input0, align 1
+ %v1 = load <32 x i8>, ptr %input1, align 1
+ %v2 = load <32 x i8>, ptr %input2, align 1
+ %interleave = call <96 x i8> @llvm.vector.interleave3.v96i8(<32 x i8> %v0, <32 x i8> %v1, <32 x i8> %v2)
+ store <96 x i8> %interleave, ptr %output, align 1
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i8:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+; CHECK-IAENABLED-NEXT: st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i8:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT: zip2 v16.16b, v0.16b, v2.16b
+; CHECK-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v2.16b
+; CHECK-IADISABLED-NEXT: zip2 v2.16b, v5.16b, v7.16b
+; CHECK-IADISABLED-NEXT: zip2 v17.16b, v3.16b, v1.16b
+; CHECK-IADISABLED-NEXT: zip1 v1.16b, v3.16b, v1.16b
+; CHECK-IADISABLED-NEXT: zip2 v3.16b, v4.16b, v6.16b
+; CHECK-IADISABLED-NEXT: zip1 v5.16b, v5.16b, v7.16b
+; CHECK-IADISABLED-NEXT: zip1 v4.16b, v4.16b, v6.16b
+; CHECK-IADISABLED-NEXT: zip1 v18.16b, v17.16b, v16.16b
+; CHECK-IADISABLED-NEXT: zip2 v16.16b, v17.16b, v16.16b
+; CHECK-IADISABLED-NEXT: zip1 v6.16b, v1.16b, v0.16b
+; CHECK-IADISABLED-NEXT: zip2 v0.16b, v1.16b, v0.16b
+; CHECK-IADISABLED-NEXT: zip1 v1.16b, v3.16b, v2.16b
+; CHECK-IADISABLED-NEXT: zip2 v2.16b, v3.16b, v2.16b
+; CHECK-IADISABLED-NEXT: zip1 v3.16b, v4.16b, v5.16b
+; CHECK-IADISABLED-NEXT: zip2 v4.16b, v4.16b, v5.16b
+; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <32 x i8>, ptr %input0, align 1
+ %v1 = load <32 x i8>, ptr %input1, align 1
+ %v2 = load <32 x i8>, ptr %input2, align 1
+ %v3 = load <32 x i8>, ptr %input3, align 1
+ %interleave = call <128 x i8> @llvm.vector.interleave4.v128i8(<32 x i8> %v0, <32 x i8> %v1, <32 x i8> %v2, <32 x i8> %v3)
+ store <128 x i8> %interleave, ptr %output, align 1
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i16:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
+; CHECK-IAENABLED-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i16:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #96
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #48
+; CHECK-IADISABLED-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x8]
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT: add sp, sp, #96
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <16 x i16>, ptr %input0, align 2
+ %v1 = load <16 x i16>, ptr %input1, align 2
+ %v2 = load <16 x i16>, ptr %input2, align 2
+ %interleave = call <48 x i16> @llvm.vector.interleave3.v48i16(<16 x i16> %v0, <16 x i16> %v1, <16 x i16> %v2)
+ store <48 x i16> %interleave, ptr %output, align 2
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i16:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
+; CHECK-IAENABLED-NEXT: st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i16:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT: zip2 v16.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT: zip2 v2.8h, v5.8h, v7.8h
+; CHECK-IADISABLED-NEXT: zip2 v17.8h, v3.8h, v1.8h
+; CHECK-IADISABLED-NEXT: zip1 v1.8h, v3.8h, v1.8h
+; CHECK-IADISABLED-NEXT: zip2 v3.8h, v4.8h, v6.8h
+; CHECK-IADISABLED-NEXT: zip1 v5.8h, v5.8h, v7.8h
+; CHECK-IADISABLED-NEXT: zip1 v4.8h, v4.8h, v6.8h
+; CHECK-IADISABLED-NEXT: zip1 v18.8h, v17.8h, v16.8h
+; CHECK-IADISABLED-NEXT: zip2 v16.8h, v17.8h, v16.8h
+; CHECK-IADISABLED-NEXT: zip1 v6.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT: zip2 v0.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT: zip1 v1.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT: zip2 v2.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT: zip1 v3.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: zip2 v4.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <16 x i16>, ptr %input0, align 2
+ %v1 = load <16 x i16>, ptr %input1, align 2
+ %v2 = load <16 x i16>, ptr %input2, align 2
+ %v3 = load <16 x i16>, ptr %input3, align 2
+ %interleave = call <64 x i16> @llvm.vector.interleave4.v64i16(<16 x i16> %v0, <16 x i16> %v1, <16 x i16> %v2, <16 x i16> %v3)
+ store <64 x i16> %interleave, ptr %output, align 2
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i64:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IAENABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; CHECK-IAENABLED-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i64:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #96
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
+; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
+; CHECK-IADISABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #48
+; CHECK-IADISABLED-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x8]
+; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
+; CHECK-IADISABLED-NEXT: add sp, sp, #96
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <4 x i64>, ptr %input0, align 8
+ %v1 = load <4 x i64>, ptr %input1, align 8
+ %v2 = load <4 x i64>, ptr %input2, align 8
+ %interleave = call <12 x i64> @llvm.vector.interleave3.v12i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2)
+ store <12 x i64> %interleave, ptr %output, align 8
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i64:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
+; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
+; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
+; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
+; CHECK-IAENABLED-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
+; CHECK-IAENABLED-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i64:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
+; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
+; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
+; CHECK-IADISABLED-NEXT: zip2 v16.2d, v0.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip2 v2.2d, v5.2d, v7.2d
+; CHECK-IADISABLED-NEXT: zip2 v17.2d, v3.2d, v1.2d
+; CHECK-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v1.2d
+; CHECK-IADISABLED-NEXT: zip2 v3.2d, v4.2d, v6.2d
+; CHECK-IADISABLED-NEXT: zip1 v5.2d, v5.2d, v7.2d
+; CHECK-IADISABLED-NEXT: zip1 v4.2d, v4.2d, v6.2d
+; CHECK-IADISABLED-NEXT: zip1 v18.2d, v17.2d, v16.2d
+; CHECK-IADISABLED-NEXT: zip2 v16.2d, v17.2d, v16.2d
+; CHECK-IADISABLED-NEXT: zip1 v6.2d, v1.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; CHECK-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip2 v2.2d, v3.2d, v2.2d
+; CHECK-IADISABLED-NEXT: zip1 v3.2d, v4.2d, v5.2d
+; CHECK-IADISABLED-NEXT: zip2 v4.2d, v4.2d, v5.2d
+; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
+; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
+; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = load <4 x i64>, ptr %input0, align 8
+ %v1 = load <4 x i64>, ptr %input1, align 8
+ %v2 = load <4 x i64>, ptr %input2, align 8
+ %v3 = load <4 x i64>, ptr %input3, align 8
+ %interleave = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleave, ptr %output, align 8
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i32_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s1, w5
+; CHECK-NEXT: fmov s3, w1
+; CHECK-NEXT: ldr s0, [sp, #56]
+; CHECK-NEXT: add x8, sp, #24
+; CHECK-NEXT: ldr s2, [sp]
+; CHECK-NEXT: add x9, sp, #16
+; CHECK-NEXT: mov v0.s[1], w4
+; CHECK-NEXT: ldr s4, [sp, #32]
+; CHECK-NEXT: ld1 { v1.s }[1], [x8]
+; CHECK-NEXT: add x8, sp, #48
+; CHECK-NEXT: mov v3.s[1], w7
+; CHECK-NEXT: ld1 { v2.s }[1], [x8]
+; CHECK-NEXT: add x8, sp, #72
+; CHECK-NEXT: ld1 { v1.s }[2], [x8]
+; CHECK-NEXT: add x8, sp, #40
+; CHECK-NEXT: ld1 { v0.s }[2], [x9]
+; CHECK-NEXT: mov v2.s[2], w3
+; CHECK-NEXT: ld1 { v3.s }[2], [x8]
+; CHECK-NEXT: add x8, sp, #64
+; CHECK-NEXT: add x9, sp, #80
+; CHECK-NEXT: mov v1.s[3], w6
+; CHECK-NEXT: ld1 { v0.s }[3], [x8]
+; CHECK-NEXT: add x8, sp, #8
+; CHECK-NEXT: mov v3.s[3], w2
+; CHECK-NEXT: ld1 { v4.s }[1], [x9]
+; CHECK-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-NEXT: str d4, [x0, #64]
+; CHECK-NEXT: stp q3, q2, [x0]
+; CHECK-NEXT: stp q0, q1, [x0, #32]
+; CHECK-NEXT: ret
+ %interleave = call <18 x i32> @llvm.vector.interleave3.v18i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2)
+ store <18 x i32> %interleave, ptr %output, align 4
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i32_dreg(ptr %output, <6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i32_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s5, w6
+; CHECK-NEXT: fmov s4, w5
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: fmov s1, w2
+; CHECK-NEXT: fmov s3, w4
+; CHECK-NEXT: mov x10, sp
+; CHECK-NEXT: fmov s2, w3
+; CHECK-NEXT: fmov s0, w1
+; CHECK-NEXT: add x11, sp, #24
+; CHECK-NEXT: ld1 { v5.s }[1], [x9]
+; CHECK-NEXT: add x9, sp, #16
+; CHECK-NEXT: ld1 { v4.s }[1], [x11]
+; CHECK-NEXT: ld1 { v1.s }[1], [x10]
+; CHECK-NEXT: add x10, sp, #8
+; CHECK-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-NEXT: mov v0.s[1], w7
+; CHECK-NEXT: ld1 { v2.s }[1], [x10]
+; CHECK-NEXT: add x10, sp, #72
+; CHECK-NEXT: add x9, sp, #80
+; CHECK-NEXT: add x11, sp, #64
+; CHECK-NEXT: ld1 { v4.s }[2], [x10]
+; CHECK-NEXT: add x10, sp, #56
+; CHECK-NEXT: add x8, sp, #40
+; CHECK-NEXT: ld1 { v5.s }[2], [x9]
+; CHECK-NEXT: add x9, sp, #48
+; CHECK-NEXT: ld1 { v3.s }[2], [x11]
+; CHECK-NEXT: ld1 { v2.s }[2], [x10]
+; CHECK-NEXT: ld1 { v1.s }[2], [x9]
+; CHECK-NEXT: ld1 { v0.s }[2], [x8]
+; CHECK-NEXT: add x9, sp, #112
+; CHECK-NEXT: add x8, sp, #104
+; CHECK-NEXT: add x11, sp, #128
+; CHECK-NEXT: add x10, sp, #120
+; CHECK-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-NEXT: add x9, sp, #96
+; CHECK-NEXT: ld1 { v2.s }[3], [x8]
+; CHECK-NEXT: add x8, sp, #88
+; CHECK-NEXT: ld1 { v5.s }[3], [x11]
+; CHECK-NEXT: ld1 { v4.s }[3], [x10]
+; CHECK-NEXT: ld1 { v1.s }[3], [x9]
+; CHECK-NEXT: ld1 { v0.s }[3], [x8]
+; CHECK-NEXT: stp q2, q3, [x0, #32]
+; CHECK-NEXT: str q5, [x0, #80]
+; CHECK-NEXT: str q4, [x0, #64]
+; CHECK-NEXT: stp q0, q1, [x0]
+; CHECK-NEXT: ret
+ %interleave = call <24 x i32> @llvm.vector.interleave4.v24i32(<6 x i32> %v0, <6 x i32> %v1, <6 x i32> %v2, <6 x i32> %v3)
+ store <24 x i32> %interleave, ptr %output, align 4
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i16_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #192
+; CHECK-NEXT: .cfi_def_cfa_offset 192
+; CHECK-NEXT: ldr q0, [x1]
+; CHECK-NEXT: ldr d3, [x1, #16]
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: ldr q1, [x2]
+; CHECK-NEXT: ldr d4, [x2, #16]
+; CHECK-NEXT: ldr q2, [x3]
+; CHECK-NEXT: ldr d5, [x3, #16]
+; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
+; CHECK-NEXT: add x8, sp, #48
+; CHECK-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x8]
+; CHECK-NEXT: ldr q0, [sp, #32]
+; CHECK-NEXT: dup v1.2d, v0.d[1]
+; CHECK-NEXT: str d1, [sp, #144]
+; CHECK-NEXT: ldr q1, [sp, #64]
+; CHECK-NEXT: ldr q3, [sp, #144]
+; CHECK-NEXT: str d1, [sp, #112]
+; CHECK-NEXT: ldp q2, q1, [sp]
+; CHECK-NEXT: ext v0.16b, v1.16b, v0.16b, #8
+; CHECK-NEXT: mov v1.d[1], v0.d[0]
+; CHECK-NEXT: ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-NEXT: ldr q3, [sp, #48]
+; CHECK-NEXT: stp q2, q1, [x0]
+; CHECK-NEXT: stp q0, q3, [x0, #32]
+; CHECK-NEXT: ldr q0, [sp, #112]
+; CHECK-NEXT: str d0, [x0, #64]
+; CHECK-NEXT: add sp, sp, #192
+; CHECK-NEXT: ret
+ %v0 = load <12 x i16>, ptr %input0, align 2
+ %v1 = load <12 x i16>, ptr %input1, align 2
+ %v2 = load <12 x i16>, ptr %input2, align 2
+ %interleave = call <36 x i16> @llvm.vector.interleave3.v36i16(<12 x i16> %v0, <12 x i16> %v1, <12 x i16> %v2)
+ store <36 x i16> %interleave, ptr %output, align 2
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i16_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i16_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #128
+; CHECK-NEXT: .cfi_def_cfa_offset 128
+; CHECK-NEXT: ldr q0, [x2]
+; CHECK-NEXT: ldr d1, [x1, #16]
+; CHECK-NEXT: ldr d2, [x2, #16]
+; CHECK-NEXT: ldr d3, [x3, #16]
+; CHECK-NEXT: ldr q4, [x4]
+; CHECK-NEXT: ldr d5, [x4, #16]
+; CHECK-NEXT: ldr q6, [x1]
+; CHECK-NEXT: ldr q7, [x3]
+; CHECK-NEXT: zip1 v2.8h, v2.8h, v5.8h
+; CHECK-NEXT: zip1 v1.8h, v1.8h, v3.8h
+; CHECK-NEXT: zip2 v3.8h, v0.8h, v4.8h
+; CHECK-NEXT: zip2 v5.8h, v6.8h, v7.8h
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v4.8h
+; CHECK-NEXT: zip1 v4.8h, v6.8h, v7.8h
+; CHECK-NEXT: zip2 v6.8h, v1.8h, v2.8h
+; CHECK-NEXT: zip1 v1.8h, v1.8h, v2.8h
+; CHECK-NEXT: zip1 v7.8h, v5.8h, v3.8h
+; CHECK-NEXT: zip2 v16.8h, v4.8h, v0.8h
+; CHECK-NEXT: zip1 v0.8h, v4.8h, v0.8h
+; CHECK-NEXT: zip2 v3.8h, v5.8h, v3.8h
+; CHECK-NEXT: dup v2.2d, v6.d[1]
+; CHECK-NEXT: dup v17.2d, v7.d[1]
+; CHECK-NEXT: ext v7.16b, v16.16b, v7.16b, #8
+; CHECK-NEXT: str d2, [sp, #16]
+; CHECK-NEXT: ext v2.16b, v1.16b, v6.16b, #8
+; CHECK-NEXT: str d17, [sp, #80]
+; CHECK-NEXT: ldr q5, [sp, #16]
+; CHECK-NEXT: mov v16.d[1], v7.d[0]
+; CHECK-NEXT: ldr q4, [sp, #80]
+; CHECK-NEXT: ext v4.16b, v7.16b, v4.16b, #8
+; CHECK-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NEXT: stp q0, q16, [x0]
+; CHECK-NEXT: ext v0.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT: stp q4, q3, [x0, #32]
+; CHECK-NEXT: stp q1, q0, [x0, #64]
+; CHECK-NEXT: add sp, sp, #128
+; CHECK-NEXT: ret
+ %v0 = load <12 x i16>, ptr %input0, align 2
+ %v1 = load <12 x i16>, ptr %input1, align 2
+ %v2 = load <12 x i16>, ptr %input2, align 2
+ %v3 = load <12 x i16>, ptr %input3, align 2
+ %interleave = call <48 x i16> @llvm.vector.interleave4.v48i16(<12 x i16> %v0, <12 x i16> %v1, <12 x i16> %v2, <12 x i16> %v3)
+ store <48 x i16> %interleave, ptr %output, align 2
+ ret void
+}
+
+define void @wide_vector_interleave_st3_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
+; CHECK-LABEL: wide_vector_interleave_st3_i8_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #192
+; CHECK-NEXT: .cfi_def_cfa_offset 192
+; CHECK-NEXT: ldr q0, [x1]
+; CHECK-NEXT: ldr d3, [x1, #16]
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: ldr q1, [x2]
+; CHECK-NEXT: ldr d4, [x2, #16]
+; CHECK-NEXT: ldr q2, [x3]
+; CHECK-NEXT: ldr d5, [x3, #16]
+; CHECK-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x8]
+; CHECK-NEXT: add x8, sp, #48
+; CHECK-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x8]
+; CHECK-NEXT: ldr q0, [sp, #32]
+; CHECK-NEXT: dup v1.2d, v0.d[1]
+; CHECK-NEXT: str d1, [sp, #144]
+; CHECK-NEXT: ldr q1, [sp, #64]
+; CHECK-NEXT: ldr q3, [sp, #144]
+; CHECK-NEXT: str d1, [sp, #112]
+; CHECK-NEXT: ldp q2, q1, [sp]
+; CHECK-NEXT: ext v0.16b, v1.16b, v0.16b, #8
+; CHECK-NEXT: mov v1.d[1], v0.d[0]
+; CHECK-NEXT: ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-NEXT: ldr q3, [sp, #48]
+; CHECK-NEXT: stp q2, q1, [x0]
+; CHECK-NEXT: stp q0, q3, [x0, #32]
+; CHECK-NEXT: ldr q0, [sp, #112]
+; CHECK-NEXT: str d0, [x0, #64]
+; CHECK-NEXT: add sp, sp, #192
+; CHECK-NEXT: ret
+ %v0 = load <24 x i8>, ptr %input0, align 1
+ %v1 = load <24 x i8>, ptr %input1, align 1
+ %v2 = load <24 x i8>, ptr %input2, align 1
+ %interleave = call <72 x i8> @llvm.vector.interleave3.v72i8(<24 x i8> %v0, <24 x i8> %v1, <24 x i8> %v2)
+ store <72 x i8> %interleave, ptr %output, align 1
+ ret void
+}
+
+define void @wide_vector_interleave_st4_i8_dreg(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
+; CHECK-LABEL: wide_vector_interleave_st4_i8_dreg:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #128
+; CHECK-NEXT: .cfi_def_cfa_offset 128
+; CHECK-NEXT: ldr q0, [x2]
+; CHECK-NEXT: ldr d1, [x1, #16]
+; CHECK-NEXT: ldr d2, [x2, #16]
+; CHECK-NEXT: ldr d3, [x3, #16]
+; CHECK-NEXT: ldr q4, [x4]
+; CHECK-NEXT: ldr d5, [x4, #16]
+; CHECK-NEXT: ldr q6, [x1]
+; CHECK-NEXT: ldr q7, [x3]
+; CHECK-NEXT: zip1 v2.16b, v2.16b, v5.16b
+; CHECK-NEXT: zip1 v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: zip2 v3.16b, v0.16b, v4.16b
+; CHECK-NEXT: zip2 v5.16b, v6.16b, v7.16b
+; CHECK-NEXT: zip1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT: zip1 v4.16b, v6.16b, v7.16b
+; CHECK-NEXT: zip2 v6.16b, v1.16b, v2.16b
+; CHECK-NEXT: zip1 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: zip1 v7.16b, v5.16b, v3.16b
+; CHECK-NEXT: zip2 v16.16b, v4.16b, v0.16b
+; CHECK-NEXT: zip1 v0.16b, v4.16b, v0.16b
+; CHECK-NEXT: zip2 v3.16b, v5.16b, v3.16b
+; CHECK-NEXT: dup v2.2d, v6.d[1]
+; CHECK-NEXT: dup v17.2d, v7.d[1]
+; CHECK-NEXT: ext v7.16b, v16.16b, v7.16b, #8
+; CHECK-NEXT: str d2, [sp, #16]
+; CHECK-NEXT: ext v2.16b, v1.16b, v6.16b, #8
+; CHECK-NEXT: str d17, [sp, #80]
+; CHECK-NEXT: ldr q5, [sp, #16]
+; CHECK-NEXT: mov v16.d[1], v7.d[0]
+; CHECK-NEXT: ldr q4, [sp, #80]
+; CHECK-NEXT: ext v4.16b, v7.16b, v4.16b, #8
+; CHECK-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-NEXT: stp q0, q16, [x0]
+; CHECK-NEXT: ext v0.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT: stp q4, q3, [x0, #32]
+; CHECK-NEXT: stp q1, q0, [x0, #64]
+; CHECK-NEXT: add sp, sp, #128
+; CHECK-NEXT: ret
+ %v0 = load <24 x i8>, ptr %input0, align 1
+ %v1 = load <24 x i8>, ptr %input1, align 1
+ %v2 = load <24 x i8>, ptr %input2, align 1
+ %v3 = load <24 x i8>, ptr %input3, align 1
+ %interleave = call <96 x i8> @llvm.vector.interleave4.v96i8(<24 x i8> %v0, <24 x i8> %v1, <24 x i8> %v2, <24 x i8> %v3)
+ store <96 x i8> %interleave, ptr %output, align 1
+ ret void
+}
>From 1dcd2c2b4f5acb2018bd5e5dbf18fe956a6bbcae Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 2 Sep 2026 17:35:57 +0100
Subject: [PATCH 2/2] perform the dag combine after the type legalization
---
llvm/include/llvm/CodeGen/SelectionDAG.h | 14 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 18 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 20 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 76 +++-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 4 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 13 +-
.../CodeGen/AArch64/interleaved-accesses.ll | 95 ++---
.../AArch64/vector-interleave-store.ll | 328 ++++--------------
8 files changed, 203 insertions(+), 365 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index e2ad79c4ca931..a600012638cd8 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2603,12 +2603,14 @@ class SelectionDAG {
LLVM_ABI std::pair<SDValue, SDValue>
UnrollVectorOverflowOp(SDNode *N, unsigned ResNE = 0);
- /// Return true if loads are next to each other and can be
- /// merged. Check that both are nonvolatile and if LD is loading
- /// 'Bytes' bytes from a location that is 'Dist' units away from the
- /// location that the 'Base' load is loading from.
- LLVM_ABI bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
- unsigned Bytes, int Dist) const;
+ /// Return true if load/store nodes are next to each other and can be merged.
+ /// Check that both are nonvolatile and if \p LS is accessing \p Bytes bytes
+ /// at a location that is \p Dist units away from the location accessed by
+ /// \p Base.
+ LLVM_ABI bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
+ LSBaseSDNode *Base,
+ unsigned Bytes,
+ int Dist) const;
/// Infer alignment of a load / store address. Return std::nullopt if it
/// cannot be inferred.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 09e26fe2aa58c..a34170eb7c88d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -12146,7 +12146,8 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
LHS->getAddressSpace() == RHS->getAddressSpace() &&
(LHS->hasNUsesOfValue(1, 0) || RHS->hasNUsesOfValue(1, 0)) &&
ISD::isNON_EXTLoad(RHS) && ISD::isNON_EXTLoad(LHS)) {
- if (DAG.areNonVolatileConsecutiveLoads(LHS, RHS, BitWidth / 8, 1)) {
+ if (DAG.areNonVolatileConsecutiveLoadsOrStores(LHS, RHS, BitWidth / 8,
+ 1)) {
SDLoc DL(RHS);
uint64_t PtrOff =
IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8);
@@ -18136,9 +18137,10 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
EVT LD1VT = LD1->getValueType(0);
unsigned LD1Bytes = LD1VT.getStoreSize();
if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
- DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1) &&
+ DAG.areNonVolatileConsecutiveLoadsOrStores(LD2, LD1, LD1Bytes, 1) &&
TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
- *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
+ *LD1->getMemOperand(), &LD1Fast) &&
+ LD1Fast)
return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
LD1->getPointerInfo(), LD1->getAlign());
@@ -25217,11 +25219,11 @@ SDValue DAGCombiner::combineInsertEltToLoad(SDNode *N, unsigned InsIndex) {
// Check that the offset between the pointers to produce a single continuous
// load.
if (InsIndex == 0) {
- if (!DAG.areNonVolatileConsecutiveLoads(ScalarLoad, VecLoad, EltSize / 8,
- -1))
+ if (!DAG.areNonVolatileConsecutiveLoadsOrStores(ScalarLoad, VecLoad,
+ EltSize / 8, -1))
return SDValue();
} else {
- if (!DAG.areNonVolatileConsecutiveLoads(
+ if (!DAG.areNonVolatileConsecutiveLoadsOrStores(
VecLoad, ScalarLoad, VT.getVectorNumElements() * EltSize / 8, -1))
return SDValue();
}
@@ -27530,10 +27532,10 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Check if the loads are consecutive.
LoadSDNode *Base = nullptr;
- if (DAG.areNonVolatileConsecutiveLoads(
+ if (DAG.areNonVolatileConsecutiveLoadsOrStores(
LoadB, LoadA, LoadB->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
Base = LoadA;
- } else if (DAG.areNonVolatileConsecutiveLoads(
+ } else if (DAG.areNonVolatileConsecutiveLoadsOrStores(
LoadA, LoadB, LoadA->getMemoryVT().getStoreSize(),
/*Dist=*/1)) {
Base = LoadB;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 00cef99069347..1139b0337b60d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -14583,25 +14583,25 @@ std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
getBuildVector(NewOvVT, dl, OvScalars));
}
-bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
- LoadSDNode *Base,
- unsigned Bytes,
- int Dist) const {
- if (LD->isVolatile() || Base->isVolatile())
+bool SelectionDAG::areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
+ LSBaseSDNode *Base,
+ unsigned Bytes,
+ int Dist) const {
+ if (LS->isVolatile() || Base->isVolatile())
return false;
// TODO: probably too restrictive for atomics, revisit
- if (!LD->isSimple())
+ if (!LS->isSimple())
return false;
- if (LD->isIndexed() || Base->isIndexed())
+ if (LS->isIndexed() || Base->isIndexed())
return false;
- if (LD->getChain() != Base->getChain())
+ if (LS->getChain() != Base->getChain())
return false;
- EVT VT = LD->getMemoryVT();
+ EVT VT = LS->getMemoryVT();
if (VT.getSizeInBits() / 8 != Bytes)
return false;
auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
- auto LocDecomp = BaseIndexOffset::match(LD, *this);
+ auto LocDecomp = BaseIndexOffset::match(LS, *this);
int64_t Offset = 0;
if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 36a86241fe3b2..3bdf278218a90 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -24224,8 +24224,8 @@ static bool areLoadedOffsetButOtherwiseSame(SDValue Op0, SDValue Op1,
all_of(zip(Loads0, Loads1), [&DAG](auto L) {
unsigned Size = get<0>(L)->getValueType(0).getSizeInBits();
return Size == get<1>(L)->getValueType(0).getSizeInBits() &&
- DAG.areNonVolatileConsecutiveLoads(get<1>(L), get<0>(L),
- Size / 8, 1);
+ DAG.areNonVolatileConsecutiveLoadsOrStores(
+ get<1>(L), get<0>(L), Size / 8, 1);
});
}
@@ -27710,6 +27710,9 @@ static SDValue
performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG);
+static SDValue performLegalizedInterleavedStoreCombine(
+ StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);
+
static SDValue performSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -27744,6 +27747,9 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG))
return Res;
+ if (SDValue Res = performLegalizedInterleavedStoreCombine(ST, DCI, DAG))
+ return Res;
+
// Cast ptr32 and ptr64 pointers to the default address space before a store.
unsigned AddrSpace = ST->getAddressSpace();
if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR ||
@@ -28103,6 +28109,72 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
MemN->getMemoryVT(), MemN->getMemOperand());
}
+static SDValue performLegalizedInterleavedStoreCombine(
+ StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
+ // Type legalization splits a wide interleaved store into consecutive legal
+ // stores. Combine each group that directly stores all results of a legal
+ // VECTOR_INTERLEAVE into a structured store.
+ if (DCI.getDAGCombineLevel() != AfterLegalizeTypes ||
+ !DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable())
+ return SDValue();
+
+ SDValue StoredValue = ST->getValue();
+ SDNode *Interleave = StoredValue.getNode();
+ if (Interleave->getOpcode() != ISD::VECTOR_INTERLEAVE)
+ return SDValue();
+
+ unsigned NumParts = Interleave->getNumOperands();
+ if (NumParts < 2 || NumParts > 4)
+ return SDValue();
+
+ EVT SubVecTy = Interleave->getValueType(0);
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (!SubVecTy.isFixedLengthVector() || !TLI.isTypeLegal(SubVecTy) ||
+ (!SubVecTy.is64BitVector() && !SubVecTy.is128BitVector()))
+ return SDValue();
+
+ SmallVector<StoreSDNode *, 4> Stores(NumParts, nullptr);
+ for (SDUse &Use : Interleave->uses()) {
+ unsigned ResNo = Use.getResNo();
+ auto *Store = dyn_cast<StoreSDNode>(Use.getUser());
+ if (Stores[ResNo] || !Store)
+ return SDValue();
+ Stores[ResNo] = Store;
+ }
+
+ StoreSDNode *BaseStore = Stores[0];
+ unsigned Bytes = SubVecTy.getStoreSize().getFixedValue();
+ for (auto [Idx, Store] : enumerate(Stores)) {
+ if (!DAG.areNonVolatileConsecutiveLoadsOrStores(Store, BaseStore, Bytes,
+ Idx))
+ return SDValue();
+ }
+
+ static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
+ Intrinsic::aarch64_neon_st3,
+ Intrinsic::aarch64_neon_st4};
+ SDLoc DL(Interleave);
+ SmallVector<SDValue, 8> Ops = {
+ BaseStore->getChain(),
+ DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64)};
+ Ops.append(Interleave->op_begin(), Interleave->op_end());
+ Ops.push_back(BaseStore->getBasePtr());
+
+ EVT MemVT = EVT::getVectorVT(
+ *DAG.getContext(), SubVecTy.getVectorElementType(),
+ SubVecTy.getVectorElementCount().multiplyCoefficientBy(NumParts));
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(BaseStore->getMemOperand(), 0, NumParts * Bytes);
+ SDValue NewStore = DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
+
+ for (StoreSDNode *Store : Stores)
+ if (Store != ST)
+ DCI.CombineTo(Store, NewStore);
+ return NewStore;
+}
+
static SDValue performMSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index d599bba54079b..3866a9831a7b3 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -16554,9 +16554,9 @@ static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
// We only care about regular loads. The PPC-specific load intrinsics
// will not lead to a merge opportunity.
- if (!DAG.areNonVolatileConsecutiveLoads(LD2, LD1, ElemSize, 1))
+ if (!DAG.areNonVolatileConsecutiveLoadsOrStores(LD2, LD1, ElemSize, 1))
InputsAreConsecutiveLoads = false;
- if (!DAG.areNonVolatileConsecutiveLoads(LD1, LD2, ElemSize, 1))
+ if (!DAG.areNonVolatileConsecutiveLoadsOrStores(LD1, LD2, ElemSize, 1))
InputsAreReverseConsecutive = false;
// Exit early if the loads are neither consecutive nor reverse consecutive.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6144e1f0bf99d..a14ab818a157c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -7910,7 +7910,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
Loads[BaseIdx] == Ld && ByteOffsets[BaseIdx] == 0);
}
int Stride = EltIdx - FirstLoadedElt;
- if (DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes, Stride))
+ if (DAG.areNonVolatileConsecutiveLoadsOrStores(Ld, Base, BaseSizeInBytes,
+ Stride))
return true;
// Try again using the memory load size (we might have broken a large load
// into smaller elements), ensure the stride is the full memory load size
@@ -7919,8 +7920,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
if (((Stride * BaseSizeInBits) % BaseMemSizeInBits) == 0 &&
(BaseMemSizeInBits % BaseSizeInBits) == 0) {
unsigned Scale = BaseMemSizeInBits / BaseSizeInBits;
- return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseMemSizeInBits / 8,
- Stride / Scale);
+ return DAG.areNonVolatileConsecutiveLoadsOrStores(
+ Ld, Base, BaseMemSizeInBits / 8, Stride / Scale);
}
return false;
};
@@ -8126,7 +8127,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
bool AllMatch = true;
for (unsigned K = 1; K < NumElems && AllMatch; ++K) {
AllMatch = AllMatch && ByteOffsets[K] == 0 &&
- DAG.areNonVolatileConsecutiveLoads(
+ DAG.areNonVolatileConsecutiveLoadsOrStores(
Loads[K], LDBase, BaseSizeInBytes, K * LaneStride);
}
if (AllMatch) {
@@ -62060,7 +62061,7 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
auto *VecLd = dyn_cast<LoadSDNode>(Vec);
auto *SubLd = dyn_cast<LoadSDNode>(SubVec);
if (VecLd && SubLd &&
- DAG.areNonVolatileConsecutiveLoads(
+ DAG.areNonVolatileConsecutiveLoadsOrStores(
SubLd, VecLd, SubVec.getValueSizeInBits() / 8, 0)) {
SDValue BcastLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, dl, OpVT,
SubVecVT, SubLd, 0, DAG);
@@ -62096,7 +62097,7 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
// Match insertion of subvector load that perfectly aliases a base load.
if ((IdxVal % SubVecNumElts) == 0 && ISD::isNormalLoad(Vec.getNode()) &&
ISD::isNormalLoad(SubVec.getNode()) &&
- DAG.areNonVolatileConsecutiveLoads(
+ DAG.areNonVolatileConsecutiveLoadsOrStores(
cast<LoadSDNode>(SubVec), cast<LoadSDNode>(Vec),
SubVec.getValueSizeInBits() / 8, IdxVal / SubVecNumElts))
return Vec;
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index bedce9ec6a86e..2e52b3b373484 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -2331,40 +2331,17 @@ define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_factor3_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-IAENABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT: mov v4.16b, v3.16b
-; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
-; NEON-IAENABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
-; NEON-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: sub sp, sp, #96
-; NEON-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; NEON-IADISABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IADISABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IADISABLED-NEXT: mov x8, sp
-; NEON-IADISABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IADISABLED-NEXT: mov v4.16b, v3.16b
-; NEON-IADISABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IADISABLED-NEXT: mov v3.16b, v1.16b
-; NEON-IADISABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x8]
-; NEON-IADISABLED-NEXT: add x8, sp, #48
-; NEON-IADISABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x8]
-; NEON-IADISABLED-NEXT: ldp q2, q0, [sp, #64]
-; NEON-IADISABLED-NEXT: ldp q5, q1, [sp, #32]
-; NEON-IADISABLED-NEXT: ldp q3, q4, [sp]
-; NEON-IADISABLED-NEXT: stp q2, q0, [x0, #64]
-; NEON-IADISABLED-NEXT: stp q5, q1, [x0, #32]
-; NEON-IADISABLED-NEXT: stp q3, q4, [x0]
-; NEON-IADISABLED-NEXT: add sp, sp, #96
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_factor3_wide_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v4.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_factor3_wide_intrinsic:
; NO_NEON: // %bb.0:
@@ -2458,43 +2435,19 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-IAENABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
-; NEON-IAENABLED-NEXT: mov v19.16b, v6.16b
-; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT: mov v6.16b, v5.16b
-; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT: mov v5.16b, v3.16b
-; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
-; NEON-IAENABLED-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
-; NEON-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v16.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT: zip2 v17.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip1 v3.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip2 v5.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT: zip1 v2.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT: zip2 v18.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT: zip1 v16.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT: zip2 v3.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip1 v5.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip2 v6.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT: stp q16, q18, [x0, #96]
-; NEON-IADISABLED-NEXT: stp q1, q4, [x0, #64]
-; NEON-IADISABLED-NEXT: stp q0, q6, [x0]
-; NEON-IADISABLED-NEXT: stp q5, q3, [x0, #32]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_factor4_wide_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-NEXT: mov v19.16b, v6.16b
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: mov v6.16b, v5.16b
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v5.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v4.16b, v1.16b
+; NEON-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_factor4_wide_intrinsic:
; NO_NEON: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index 9732dfc246e70..a245990361f3e 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -96,34 +96,14 @@ define void @wide_vector_interleave_st2(ptr %input0, ptr %input1, ptr %output) {
}
define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-; CHECK-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: sub sp, sp, #96
-; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT: mov x8, sp
-; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
-; CHECK-IADISABLED-NEXT: add x8, sp, #48
-; CHECK-IADISABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x8]
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT: add sp, sp, #96
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q3, [x1]
+; CHECK-NEXT: ldp q1, q4, [x2]
+; CHECK-NEXT: ldp q2, q5, [x3]
+; CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-NEXT: ret
%v0 = load <8 x i32>, ptr %input0, align 4
%v1 = load <8 x i32>, ptr %input1, align 4
%v2 = load <8 x i32>, ptr %input2, align 4
@@ -133,43 +113,15 @@ define void @wide_vector_interleave_st3(ptr %output, ptr %input0, ptr %input1, p
}
define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT: zip2 v16.4s, v0.4s, v2.4s
-; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
-; CHECK-IADISABLED-NEXT: zip2 v2.4s, v5.4s, v7.4s
-; CHECK-IADISABLED-NEXT: zip2 v17.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: zip2 v3.4s, v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT: zip1 v5.4s, v5.4s, v7.4s
-; CHECK-IADISABLED-NEXT: zip1 v4.4s, v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT: zip1 v18.4s, v17.4s, v16.4s
-; CHECK-IADISABLED-NEXT: zip2 v16.4s, v17.4s, v16.4s
-; CHECK-IADISABLED-NEXT: zip1 v6.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: zip2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: zip1 v3.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: zip2 v4.4s, v4.4s, v5.4s
-; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q4, [x1]
+; CHECK-NEXT: ldp q1, q5, [x2]
+; CHECK-NEXT: ldp q2, q6, [x3]
+; CHECK-NEXT: ldp q3, q7, [x4]
+; CHECK-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-NEXT: ret
%v0 = load <8 x i32>, ptr %input0, align 4
%v1 = load <8 x i32>, ptr %input1, align 4
%v2 = load <8 x i32>, ptr %input2, align 4
@@ -180,34 +132,14 @@ define void @wide_vector_interleave_st4(ptr %output, ptr %input0, ptr %input1, p
}
define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i8:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
-; CHECK-IAENABLED-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i8:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: sub sp, sp, #96
-; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT: mov x8, sp
-; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x8]
-; CHECK-IADISABLED-NEXT: add x8, sp, #48
-; CHECK-IADISABLED-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x8]
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT: add sp, sp, #96
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st3_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q3, [x1]
+; CHECK-NEXT: ldp q1, q4, [x2]
+; CHECK-NEXT: ldp q2, q5, [x3]
+; CHECK-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x0], #48
+; CHECK-NEXT: st3 { v3.16b, v4.16b, v5.16b }, [x0]
+; CHECK-NEXT: ret
%v0 = load <32 x i8>, ptr %input0, align 1
%v1 = load <32 x i8>, ptr %input1, align 1
%v2 = load <32 x i8>, ptr %input2, align 1
@@ -217,43 +149,15 @@ define void @wide_vector_interleave_st3_i8(ptr %output, ptr %input0, ptr %input1
}
define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i8:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
-; CHECK-IAENABLED-NEXT: st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i8:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT: zip2 v16.16b, v0.16b, v2.16b
-; CHECK-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v2.16b
-; CHECK-IADISABLED-NEXT: zip2 v2.16b, v5.16b, v7.16b
-; CHECK-IADISABLED-NEXT: zip2 v17.16b, v3.16b, v1.16b
-; CHECK-IADISABLED-NEXT: zip1 v1.16b, v3.16b, v1.16b
-; CHECK-IADISABLED-NEXT: zip2 v3.16b, v4.16b, v6.16b
-; CHECK-IADISABLED-NEXT: zip1 v5.16b, v5.16b, v7.16b
-; CHECK-IADISABLED-NEXT: zip1 v4.16b, v4.16b, v6.16b
-; CHECK-IADISABLED-NEXT: zip1 v18.16b, v17.16b, v16.16b
-; CHECK-IADISABLED-NEXT: zip2 v16.16b, v17.16b, v16.16b
-; CHECK-IADISABLED-NEXT: zip1 v6.16b, v1.16b, v0.16b
-; CHECK-IADISABLED-NEXT: zip2 v0.16b, v1.16b, v0.16b
-; CHECK-IADISABLED-NEXT: zip1 v1.16b, v3.16b, v2.16b
-; CHECK-IADISABLED-NEXT: zip2 v2.16b, v3.16b, v2.16b
-; CHECK-IADISABLED-NEXT: zip1 v3.16b, v4.16b, v5.16b
-; CHECK-IADISABLED-NEXT: zip2 v4.16b, v4.16b, v5.16b
-; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st4_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q4, [x1]
+; CHECK-NEXT: ldp q1, q5, [x2]
+; CHECK-NEXT: ldp q2, q6, [x3]
+; CHECK-NEXT: ldp q3, q7, [x4]
+; CHECK-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0], #64
+; CHECK-NEXT: st4 { v4.16b, v5.16b, v6.16b, v7.16b }, [x0]
+; CHECK-NEXT: ret
%v0 = load <32 x i8>, ptr %input0, align 1
%v1 = load <32 x i8>, ptr %input1, align 1
%v2 = load <32 x i8>, ptr %input2, align 1
@@ -264,34 +168,14 @@ define void @wide_vector_interleave_st4_i8(ptr %output, ptr %input0, ptr %input1
}
define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i16:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
-; CHECK-IAENABLED-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i16:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: sub sp, sp, #96
-; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT: mov x8, sp
-; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
-; CHECK-IADISABLED-NEXT: add x8, sp, #48
-; CHECK-IADISABLED-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x8]
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT: add sp, sp, #96
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st3_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q3, [x1]
+; CHECK-NEXT: ldp q1, q4, [x2]
+; CHECK-NEXT: ldp q2, q5, [x3]
+; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0], #48
+; CHECK-NEXT: st3 { v3.8h, v4.8h, v5.8h }, [x0]
+; CHECK-NEXT: ret
%v0 = load <16 x i16>, ptr %input0, align 2
%v1 = load <16 x i16>, ptr %input1, align 2
%v2 = load <16 x i16>, ptr %input2, align 2
@@ -301,43 +185,15 @@ define void @wide_vector_interleave_st3_i16(ptr %output, ptr %input0, ptr %input
}
define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i16:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
-; CHECK-IAENABLED-NEXT: st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i16:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT: zip2 v16.8h, v0.8h, v2.8h
-; CHECK-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v2.8h
-; CHECK-IADISABLED-NEXT: zip2 v2.8h, v5.8h, v7.8h
-; CHECK-IADISABLED-NEXT: zip2 v17.8h, v3.8h, v1.8h
-; CHECK-IADISABLED-NEXT: zip1 v1.8h, v3.8h, v1.8h
-; CHECK-IADISABLED-NEXT: zip2 v3.8h, v4.8h, v6.8h
-; CHECK-IADISABLED-NEXT: zip1 v5.8h, v5.8h, v7.8h
-; CHECK-IADISABLED-NEXT: zip1 v4.8h, v4.8h, v6.8h
-; CHECK-IADISABLED-NEXT: zip1 v18.8h, v17.8h, v16.8h
-; CHECK-IADISABLED-NEXT: zip2 v16.8h, v17.8h, v16.8h
-; CHECK-IADISABLED-NEXT: zip1 v6.8h, v1.8h, v0.8h
-; CHECK-IADISABLED-NEXT: zip2 v0.8h, v1.8h, v0.8h
-; CHECK-IADISABLED-NEXT: zip1 v1.8h, v3.8h, v2.8h
-; CHECK-IADISABLED-NEXT: zip2 v2.8h, v3.8h, v2.8h
-; CHECK-IADISABLED-NEXT: zip1 v3.8h, v4.8h, v5.8h
-; CHECK-IADISABLED-NEXT: zip2 v4.8h, v4.8h, v5.8h
-; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st4_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q4, [x1]
+; CHECK-NEXT: ldp q1, q5, [x2]
+; CHECK-NEXT: ldp q2, q6, [x3]
+; CHECK-NEXT: ldp q3, q7, [x4]
+; CHECK-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0], #64
+; CHECK-NEXT: st4 { v4.8h, v5.8h, v6.8h, v7.8h }, [x0]
+; CHECK-NEXT: ret
%v0 = load <16 x i16>, ptr %input0, align 2
%v1 = load <16 x i16>, ptr %input1, align 2
%v2 = load <16 x i16>, ptr %input2, align 2
@@ -348,34 +204,14 @@ define void @wide_vector_interleave_st4_i16(ptr %output, ptr %input0, ptr %input
}
define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st3_i64:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IAENABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
-; CHECK-IAENABLED-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st3_i64:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: sub sp, sp, #96
-; CHECK-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [x1]
-; CHECK-IADISABLED-NEXT: mov x8, sp
-; CHECK-IADISABLED-NEXT: ldp q1, q4, [x2]
-; CHECK-IADISABLED-NEXT: ldp q2, q5, [x3]
-; CHECK-IADISABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x8]
-; CHECK-IADISABLED-NEXT: add x8, sp, #48
-; CHECK-IADISABLED-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x8]
-; CHECK-IADISABLED-NEXT: ldp q0, q3, [sp, #32]
-; CHECK-IADISABLED-NEXT: ldp q2, q1, [sp, #64]
-; CHECK-IADISABLED-NEXT: ldp q4, q5, [sp]
-; CHECK-IADISABLED-NEXT: stp q0, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q2, q1, [x0, #64]
-; CHECK-IADISABLED-NEXT: stp q4, q5, [x0]
-; CHECK-IADISABLED-NEXT: add sp, sp, #96
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st3_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q3, [x1]
+; CHECK-NEXT: ldp q1, q4, [x2]
+; CHECK-NEXT: ldp q2, q5, [x3]
+; CHECK-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; CHECK-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x0]
+; CHECK-NEXT: ret
%v0 = load <4 x i64>, ptr %input0, align 8
%v1 = load <4 x i64>, ptr %input1, align 8
%v2 = load <4 x i64>, ptr %input2, align 8
@@ -385,43 +221,15 @@ define void @wide_vector_interleave_st3_i64(ptr %output, ptr %input0, ptr %input
}
define void @wide_vector_interleave_st4_i64(ptr %output, ptr %input0, ptr %input1, ptr %input2, ptr %input3) {
-; CHECK-IAENABLED-LABEL: wide_vector_interleave_st4_i64:
-; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: ldp q0, q4, [x1]
-; CHECK-IAENABLED-NEXT: ldp q1, q5, [x2]
-; CHECK-IAENABLED-NEXT: ldp q2, q6, [x3]
-; CHECK-IAENABLED-NEXT: ldp q3, q7, [x4]
-; CHECK-IAENABLED-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
-; CHECK-IAENABLED-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: wide_vector_interleave_st4_i64:
-; CHECK-IADISABLED: // %bb.0:
-; CHECK-IADISABLED-NEXT: ldp q5, q0, [x2]
-; CHECK-IADISABLED-NEXT: ldp q6, q1, [x3]
-; CHECK-IADISABLED-NEXT: ldp q7, q2, [x4]
-; CHECK-IADISABLED-NEXT: ldp q4, q3, [x1]
-; CHECK-IADISABLED-NEXT: zip2 v16.2d, v0.2d, v2.2d
-; CHECK-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v2.2d
-; CHECK-IADISABLED-NEXT: zip2 v2.2d, v5.2d, v7.2d
-; CHECK-IADISABLED-NEXT: zip2 v17.2d, v3.2d, v1.2d
-; CHECK-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v1.2d
-; CHECK-IADISABLED-NEXT: zip2 v3.2d, v4.2d, v6.2d
-; CHECK-IADISABLED-NEXT: zip1 v5.2d, v5.2d, v7.2d
-; CHECK-IADISABLED-NEXT: zip1 v4.2d, v4.2d, v6.2d
-; CHECK-IADISABLED-NEXT: zip1 v18.2d, v17.2d, v16.2d
-; CHECK-IADISABLED-NEXT: zip2 v16.2d, v17.2d, v16.2d
-; CHECK-IADISABLED-NEXT: zip1 v6.2d, v1.2d, v0.2d
-; CHECK-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
-; CHECK-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v2.2d
-; CHECK-IADISABLED-NEXT: zip2 v2.2d, v3.2d, v2.2d
-; CHECK-IADISABLED-NEXT: zip1 v3.2d, v4.2d, v5.2d
-; CHECK-IADISABLED-NEXT: zip2 v4.2d, v4.2d, v5.2d
-; CHECK-IADISABLED-NEXT: stp q18, q16, [x0, #96]
-; CHECK-IADISABLED-NEXT: stp q1, q2, [x0, #32]
-; CHECK-IADISABLED-NEXT: stp q3, q4, [x0]
-; CHECK-IADISABLED-NEXT: stp q6, q0, [x0, #64]
-; CHECK-IADISABLED-NEXT: ret
+; CHECK-LABEL: wide_vector_interleave_st4_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q4, [x1]
+; CHECK-NEXT: ldp q1, q5, [x2]
+; CHECK-NEXT: ldp q2, q6, [x3]
+; CHECK-NEXT: ldp q3, q7, [x4]
+; CHECK-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0], #64
+; CHECK-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
+; CHECK-NEXT: ret
%v0 = load <4 x i64>, ptr %input0, align 8
%v1 = load <4 x i64>, ptr %input1, align 8
%v2 = load <4 x i64>, ptr %input2, align 8
More information about the llvm-commits
mailing list