[llvm] [AArch64] Update tests with interleave intrinsics (PR #212312)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 02:02:55 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/212312
>From 1ca49076d4c1c70fc52f0758cbf5d7da2c78ba81 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Tue, 28 Jul 2026 10:04:24 +0000
Subject: [PATCH] [AArch64] Update tests with interleave intrinsics
Also disabled interleave access pass for these test
---
.../aarch64-interleaved-access-w-undef.ll | 315 +-
.../AArch64/aarch64-interleaved-ld-combine.ll | 1084 +++++-
.../complex-deinterleaving-multiuses.ll | 769 +++-
.../AArch64/nontemporal-load-interleaved.ll | 3393 +++++++++++++++--
.../nontemporal-store-interleaved-optsize.ll | 297 +-
.../AArch64/nontemporal-store-interleaved.ll | 3235 ++++++++++++++--
.../CodeGen/AArch64/sve-fixed-ld2-alloca.ll | 56 +-
.../AArch64/sve-fixed-length-shuffles.ll | 53 +-
...-streaming-mode-fixed-length-ld2-alloca.ll | 207 +-
...sve-streaming-mode-fixed-length-shuffle.ll | 231 +-
llvm/test/CodeGen/AArch64/tbl-loops.ll | 1315 ++++++-
llvm/test/CodeGen/AArch64/vldn_shuffle.ll | 1494 ++++++--
llvm/test/CodeGen/AArch64/zext-shuffle.ll | 80 +-
13 files changed, 11052 insertions(+), 1477 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
index 07fbe5d7310f6..6b49ba0a65340 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-access-w-undef.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-unknown-linux-gnu"
@@ -25,18 +26,24 @@ BB:
}
define void @f_undef_15(<8 x i64> %a, ptr %dst) {
-; CHECK-LABEL: f_undef_15:
-; CHECK: // %bb.0: // %BB
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $q0_q1
-; CHECK-NEXT: mov x8, x0
-; CHECK-NEXT: mov v1.16b, v0.16b
-; CHECK-NEXT: st2 { v0.2d, v1.2d }, [x8], #32
-; CHECK-NEXT: st2 { v0.2d, v1.2d }, [x8]
-; CHECK-NEXT: add x8, x0, #64
-; CHECK-NEXT: st2 { v0.2d, v1.2d }, [x8]
-; CHECK-NEXT: add x8, x0, #96
-; CHECK-NEXT: st2 { v0.2d, v1.2d }, [x8]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: f_undef_15:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: mov v1.16b, v0.16b
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_undef_15:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: stp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: ret
BB:
%S = shufflevector <8 x i64> %a, <8 x i64> %a, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef,i32 undef, i32 undef, i32 undef, i32 undef,i32 undef, i32 undef, i32 undef, i32 undef,i32 undef, i32 undef, i32 undef, i32 undef>
store <16 x i64> %S, ptr %dst, align 64
@@ -44,24 +51,40 @@ BB:
}
define void @f_undef_1(<8 x i64> %a, ptr %dst) {
-; CHECK-LABEL: f_undef_1:
-; CHECK: // %bb.0: // %BB
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $q1_q2
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $q3_q4
-; CHECK-NEXT: mov x8, x0
-; CHECK-NEXT: mov v2.16b, v1.16b
-; CHECK-NEXT: mov v4.16b, v3.16b
-; CHECK-NEXT: mov v17.16b, v16.16b
-; CHECK-NEXT: mov v6.16b, v5.16b
-; CHECK-NEXT: st2 { v16.2d, v17.2d }, [x8], #32
-; CHECK-NEXT: st2 { v1.2d, v2.2d }, [x8]
-; CHECK-NEXT: add x8, x0, #64
-; CHECK-NEXT: st2 { v5.2d, v6.2d }, [x8]
-; CHECK-NEXT: add x8, x0, #96
-; CHECK-NEXT: st2 { v3.2d, v4.2d }, [x8]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: f_undef_1:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $q1_q2
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q3_q4
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: mov v4.16b, v3.16b
+; CHECK-IAENABLED-NEXT: mov v17.16b, v16.16b
+; CHECK-IAENABLED-NEXT: mov v6.16b, v5.16b
+; CHECK-IAENABLED-NEXT: st2 { v16.2d, v17.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v5.2d, v6.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v3.2d, v4.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_undef_1:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: dup v4.2d, v3.d[1]
+; CHECK-IADISABLED-NEXT: dup v3.2d, v3.d[0]
+; CHECK-IADISABLED-NEXT: dup v5.2d, v2.d[1]
+; CHECK-IADISABLED-NEXT: dup v2.2d, v2.d[0]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0, #96]
+; CHECK-IADISABLED-NEXT: dup v4.2d, v1.d[1]
+; CHECK-IADISABLED-NEXT: dup v1.2d, v1.d[0]
+; CHECK-IADISABLED-NEXT: dup v3.2d, v0.d[1]
+; CHECK-IADISABLED-NEXT: dup v0.2d, v0.d[0]
+; CHECK-IADISABLED-NEXT: stp q2, q5, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
BB:
%S = shufflevector <8 x i64> %a, <8 x i64> %a, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 undef, i32 14, i32 7, i32 15>
store <16 x i64> %S, ptr %dst, align 64
@@ -70,15 +93,25 @@ BB:
; noundefs and undefs should have the same results.
define void @noundefs(<8 x i32> %a, <8 x i32> %b, ptr %dst) {
-; CHECK-LABEL: noundefs:
-; CHECK: // %bb.0: // %BB
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $q2_q3
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: mov v2.16b, v1.16b
-; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
-; CHECK-NEXT: st2 { v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: noundefs:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: noundefs:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
BB:
%S = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 undef, i32 14, i32 7, i32 15>
store <16 x i32> %S, ptr %dst, align 64
@@ -86,18 +119,202 @@ BB:
}
define void @undefs(<8 x i32> %a, <8 x i32> %b, ptr %dst) {
-; CHECK-LABEL: undefs:
-; CHECK: // %bb.0: // %BB
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $q2_q3
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: mov v2.16b, v1.16b
-; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
-; CHECK-NEXT: st2 { v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: undefs:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: undefs:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: str q1, [x0, #48]
+; CHECK-IADISABLED-NEXT: str q0, [x0, #16]
+; CHECK-IADISABLED-NEXT: ret
BB:
%S = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 3, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 15>
store <16 x i32> %S, ptr %dst, align 64
ret void
}
+
+define void @f_undef_intrinsic(<8 x i64> %a, ptr %dst) {
+; CHECK-IAENABLED-LABEL: f_undef_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_undef_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> poison, <8 x i64> poison)
+ store <16 x i64> %S, ptr %dst, align 64
+ ret void
+}
+
+
+define void @f_poison_intrinsic(<8 x i64> %a, ptr %dst) {
+; CHECK-IAENABLED-LABEL: f_poison_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_poison_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> poison, <8 x i64> poison)
+ store <16 x i64> %S, ptr %dst, align 64
+ ret void
+}
+
+
+define void @f_undef_15_intrinsic(<8 x i64> %a, ptr %dst) {
+; CHECK-IAENABLED-LABEL: f_undef_15_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_undef_15_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: dup v1.2d, v0.d[1]
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %a.0 = extractelement <8 x i64> %a, i64 0
+ %even = insertelement <8 x i64> poison, i64 %a.0, i64 0
+ %S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> %even, <8 x i64> poison)
+ store <16 x i64> %S, ptr %dst, align 64
+ ret void
+}
+
+
+define void @f_undef_1_intrinsic(<8 x i64> %a, ptr %dst) {
+; CHECK-IAENABLED-LABEL: f_undef_1_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $q1_q2
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q3_q4
+; CHECK-IAENABLED-NEXT: mov x8, x0
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: mov v4.16b, v3.16b
+; CHECK-IAENABLED-NEXT: mov v17.16b, v16.16b
+; CHECK-IAENABLED-NEXT: mov v6.16b, v5.16b
+; CHECK-IAENABLED-NEXT: st2 { v16.2d, v17.2d }, [x8], #32
+; CHECK-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: st2 { v5.2d, v6.2d }, [x8]
+; CHECK-IAENABLED-NEXT: add x8, x0, #96
+; CHECK-IAENABLED-NEXT: st2 { v3.2d, v4.2d }, [x8]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: f_undef_1_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: dup v4.2d, v3.d[1]
+; CHECK-IADISABLED-NEXT: dup v3.2d, v3.d[0]
+; CHECK-IADISABLED-NEXT: dup v5.2d, v2.d[1]
+; CHECK-IADISABLED-NEXT: dup v2.2d, v2.d[0]
+; CHECK-IADISABLED-NEXT: stp q3, q4, [x0, #96]
+; CHECK-IADISABLED-NEXT: dup v4.2d, v1.d[1]
+; CHECK-IADISABLED-NEXT: dup v1.2d, v1.d[0]
+; CHECK-IADISABLED-NEXT: dup v3.2d, v0.d[1]
+; CHECK-IADISABLED-NEXT: dup v0.2d, v0.d[0]
+; CHECK-IADISABLED-NEXT: stp q2, q5, [x0, #64]
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %even = insertelement <8 x i64> %a, i64 poison, i64 6
+ %S = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> %even, <8 x i64> %a)
+ store <16 x i64> %S, ptr %dst, align 64
+ ret void
+}
+
+
+define void @noundefs_intrinsic(<8 x i32> %a, <8 x i32> %b, ptr %dst) {
+; CHECK-IAENABLED-LABEL: noundefs_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: noundefs_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %even = insertelement <8 x i32> %a, i32 poison, i64 6
+ %S = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %even, <8 x i32> %b)
+ store <16 x i32> %S, ptr %dst, align 64
+ ret void
+}
+
+
+define void @undefs_intrinsic(<8 x i32> %a, <8 x i32> %b, ptr %dst) {
+; CHECK-IAENABLED-LABEL: undefs_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %BB
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: mov v2.16b, v1.16b
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: undefs_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %BB
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
+BB:
+ %a.3 = extractelement <8 x i32> %a, i64 3
+ %a.7 = extractelement <8 x i32> %a, i64 7
+ %b.3 = extractelement <8 x i32> %b, i64 3
+ %b.7 = extractelement <8 x i32> %b, i64 7
+ %even.3 = insertelement <8 x i32> poison, i32 %a.3, i64 3
+ %even = insertelement <8 x i32> %even.3, i32 %a.7, i64 7
+ %odd.3 = insertelement <8 x i32> poison, i32 %b.3, i64 3
+ %odd = insertelement <8 x i32> %odd.3, i32 %b.7, i64 7
+ %S = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %even, <8 x i32> %odd)
+ store <16 x i32> %S, ptr %dst, align 64
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
index 35a140cb4bd40..f83739ef6f44f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
@@ -1,4 +1,6 @@
-; RUN: llc < %s | FileCheck --check-prefix AS %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck --check-prefixes=AS,AS-IAENABLED %s
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck --check-prefixes=AS,AS-IADISABLED %s
; RUN: opt -S -interleaved-load-combine < %s | FileCheck %s
; RUN: opt -S -passes=interleaved-load-combine < %s | FileCheck %s
@@ -8,22 +10,69 @@ target triple = "arm64--linux-gnu"
; This should be lowered into LD4
define void @aarch64_ilc_const(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_ilc_const(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 3
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 4
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 5
+; CHECK-NEXT: [[LOAD:%.*]] = load <16 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD3:%.*]] = load <4 x float>, ptr [[GEP3]], align 16
+; CHECK-NEXT: [[LD4:%.*]] = load <4 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[SV1:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV2:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[SV3:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV4:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE2:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT: [[M8_11:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE3:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[M12_15:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE2]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE3]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_const:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: add x8, x0, #32
+; AS-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
+; AS-IAENABLED-NEXT: stp q0, q1, [x0, #32]
+; AS-IAENABLED-NEXT: stp q2, q3, [x0, #64]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_const:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: ldp q1, q0, [x0, #64]
+; AS-IADISABLED-NEXT: ldp q4, q3, [x0, #32]
+; AS-IADISABLED-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: trn1 v17.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v5.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip1 v6.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: trn2 v16.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip2 v3.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: ext v7.16b, v1.16b, v2.16b, #8
+; AS-IADISABLED-NEXT: uzp2 v1.4s, v5.4s, v4.4s
+; AS-IADISABLED-NEXT: mov v16.d[1], v2.d[1]
+; AS-IADISABLED-NEXT: mov v3.d[1], v17.d[1]
+; AS-IADISABLED-NEXT: mov v6.d[1], v7.d[1]
+; AS-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; AS-IADISABLED-NEXT: stp q6, q16, [x0, #32]
+; AS-IADISABLED-NEXT: stp q3, q1, [x0, #64]
+; AS-IADISABLED-NEXT: ret
entry:
;;; Check LLVM transformation
-; CHECK-LABEL: @aarch64_ilc_const(
-; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, ptr %ptr, i64 2
-; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, ptr [[GEP]], align 16
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK: ret void
;;; Check if it gets lowerd
-; AS-LABEL: aarch64_ilc_const
-; AS: ld4
-; AS: ret
%gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 2
%gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 3
@@ -51,31 +100,99 @@ entry:
; This should be lowered into LD4
define void @aarch64_ilc_idx(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ADD:%.*]] = add i64 [[IDX]], 20
+; CHECK-NEXT: [[LSHR:%.*]] = lshr i64 [[ADD]], 2
+; CHECK-NEXT: [[A3:%.*]] = add i64 [[IDX]], 24
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 16
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A3]], 2
+; CHECK-NEXT: [[A4:%.*]] = add i64 [[IDX]], 28
+; CHECK-NEXT: [[IDX4:%.*]] = lshr i64 [[A4]], 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[LSHR]]
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX4]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX3]]
+; CHECK-NEXT: [[LOAD:%.*]] = load <16 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD3:%.*]] = load <4 x float>, ptr [[GEP3]], align 16
+; CHECK-NEXT: [[LD4:%.*]] = load <4 x float>, ptr [[GEP4]], align 16
+; CHECK-NEXT: [[SV1:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV2:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[SV3:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV4:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE2:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT: [[M8_11:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE3:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[M12_15:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE2]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE3]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_idx:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsl x8, x1, #2
+; AS-IAENABLED-NEXT: add x9, x8, #64
+; AS-IAENABLED-NEXT: add x11, x8, #96
+; AS-IAENABLED-NEXT: and x9, x9, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: add x10, x0, x9
+; AS-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x10]
+; AS-IAENABLED-NEXT: add x10, x8, #80
+; AS-IAENABLED-NEXT: add x8, x8, #112
+; AS-IAENABLED-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: str q0, [x0, x9]
+; AS-IAENABLED-NEXT: and x9, x11, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: str q1, [x0, x10]
+; AS-IAENABLED-NEXT: str q2, [x0, x9]
+; AS-IAENABLED-NEXT: str q3, [x0, x8]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_idx:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsl x8, x1, #2
+; AS-IADISABLED-NEXT: add x9, x8, #64
+; AS-IADISABLED-NEXT: add x10, x8, #80
+; AS-IADISABLED-NEXT: add x11, x8, #96
+; AS-IADISABLED-NEXT: and x9, x9, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x8, x8, #112
+; AS-IADISABLED-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x9, x0, x9
+; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q1, q0, [x9, #32]
+; AS-IADISABLED-NEXT: ldp q4, q3, [x9]
+; AS-IADISABLED-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: trn1 v17.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v6.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip1 v5.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: trn2 v16.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip2 v3.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: ext v7.16b, v1.16b, v2.16b, #8
+; AS-IADISABLED-NEXT: uzp2 v1.4s, v6.4s, v4.4s
+; AS-IADISABLED-NEXT: mov v16.d[1], v2.d[1]
+; AS-IADISABLED-NEXT: mov v3.d[1], v17.d[1]
+; AS-IADISABLED-NEXT: mov v5.d[1], v7.d[1]
+; AS-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; AS-IADISABLED-NEXT: str q5, [x9]
+; AS-IADISABLED-NEXT: and x9, x11, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: str q16, [x0, x10]
+; AS-IADISABLED-NEXT: str q3, [x0, x9]
+; AS-IADISABLED-NEXT: str q1, [x0, x8]
+; AS-IADISABLED-NEXT: ret
entry:
;;; Check LLVM transformation
-; CHECK-LABEL: @aarch64_ilc_idx(
-; CHECK-DAG: [[ADD:%.+]] = add i64 %idx, 16
-; CHECK-DAG: [[LSHR:%.+]] = lshr i64 [[ADD]], 2
-; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, ptr %ptr, i64 [[LSHR]]
-; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, ptr [[GEP]], align 16
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK: ret void
-
-; AS-LABEL: aarch64_ilc_idx
-; AS-DAG: lsl [[LSL:x[0-9]+]], x1, #2
-; AS-DAG: add [[ADD:x[0-9]+]], [[LSL]], #64
-; AS-DAG: and [[AND:x[0-9]+]], [[ADD]], #0xfffffffffffffff0
-; AS-DAG: add [[ADR:x[0-9]+]], x0, [[AND]]
-; AS-DAG: ld4 { v[[V0:[0-9]+]].4s, v[[V1:[0-9]+]].4s, v[[V2:[0-9]+]].4s, v[[V3:[0-9]+]].4s }, [[[ADR]]]
-; AS-DAG: str q[[V0]]
-; AS-DAG: str q[[V1]]
-; AS-DAG: str q[[V2]]
-; AS-DAG: str q[[V3]]
-; AS: ret
+
%a2 = add i64 %idx, 20
%idx2 = lshr i64 %a2, 2
@@ -113,30 +230,98 @@ entry:
; This should be lowered into LD4, a offset of has to be taken into account
%struct.ilc = type <{ float, [0 x <4 x float>] }>
define void @aarch64_ilc_struct(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_struct(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[A2:%.*]] = add i64 [[IDX]], 8
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A2]], 2
+; CHECK-NEXT: [[A3:%.*]] = add i64 [[IDX]], 12
+; CHECK-NEXT: [[IDX4:%.*]] = lshr i64 [[A3]], 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr [[STRUCT_ILC:%.*]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX2]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX3]]
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX4]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX1]]
+; CHECK-NEXT: [[LOAD:%.*]] = load <16 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[LD3:%.*]] = load <4 x float>, ptr [[GEP3]], align 4
+; CHECK-NEXT: [[LD4:%.*]] = load <4 x float>, ptr [[GEP4]], align 4
+; CHECK-NEXT: [[SV1:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV2:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[SV3:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[SV4:%.*]] = shufflevector <4 x float> [[LD3]], <4 x float> [[LD4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[SV3]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE2:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT: [[M8_11:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE3:%.*]] = shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[M12_15:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[SV4]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE2]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE3]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_struct:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsl x8, x1, #2
+; AS-IAENABLED-NEXT: add x9, x0, #4
+; AS-IAENABLED-NEXT: and x10, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: add x12, x8, #32
+; AS-IAENABLED-NEXT: add x11, x9, x10
+; AS-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x11]
+; AS-IAENABLED-NEXT: add x11, x8, #16
+; AS-IAENABLED-NEXT: add x8, x8, #48
+; AS-IAENABLED-NEXT: and x11, x11, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: str q0, [x9, x10]
+; AS-IAENABLED-NEXT: and x10, x12, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: str q1, [x9, x11]
+; AS-IAENABLED-NEXT: str q2, [x9, x10]
+; AS-IAENABLED-NEXT: str q3, [x9, x8]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_struct:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsl x8, x1, #2
+; AS-IADISABLED-NEXT: add x9, x0, #4
+; AS-IADISABLED-NEXT: and x10, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x11, x8, #16
+; AS-IADISABLED-NEXT: add x12, x8, #32
+; AS-IADISABLED-NEXT: add x10, x9, x10
+; AS-IADISABLED-NEXT: add x8, x8, #48
+; AS-IADISABLED-NEXT: and x11, x11, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q1, q0, [x10, #32]
+; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q4, q3, [x10]
+; AS-IADISABLED-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: trn1 v17.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v6.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip1 v5.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: trn2 v16.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: zip2 v3.4s, v4.4s, v3.4s
+; AS-IADISABLED-NEXT: ext v7.16b, v1.16b, v2.16b, #8
+; AS-IADISABLED-NEXT: uzp2 v1.4s, v6.4s, v4.4s
+; AS-IADISABLED-NEXT: mov v16.d[1], v2.d[1]
+; AS-IADISABLED-NEXT: mov v3.d[1], v17.d[1]
+; AS-IADISABLED-NEXT: mov v5.d[1], v7.d[1]
+; AS-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; AS-IADISABLED-NEXT: str q5, [x10]
+; AS-IADISABLED-NEXT: and x10, x12, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: str q16, [x9, x11]
+; AS-IADISABLED-NEXT: str q3, [x9, x10]
+; AS-IADISABLED-NEXT: str q1, [x9, x8]
+; AS-IADISABLED-NEXT: ret
entry:
;;; Check LLVM transformation
-; CHECK-LABEL: @aarch64_ilc_struct(
-; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2
-; CHECK-DAG: [[GEP:%.+]] = getelementptr %struct.ilc, ptr %ptr, i32 0, i32 1, i64 [[LSHR]]
-; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, ptr [[GEP]], align 4
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK: ret void
-
-; AS-LABEL: aarch64_ilc_struct
-; AS-DAG: lsl [[LSL:x[0-9]+]], x1, #2
-; AS-DAG: add [[ADD:x[0-9]+]], x0, #4
-; AS-DAG: and [[AND:x[0-9]+]], [[LSL]], #0xfffffffffffffff0
-; AS-DAG: add [[ADR:x[0-9]+]], [[ADD]], [[AND]]
-; AS-DAG: ld4 { v[[V0:[0-9]+]].4s, v[[V1:[0-9]+]].4s, v[[V2:[0-9]+]].4s, v[[V3:[0-9]+]].4s }, [[[ADR]]]
-; AS-DAG: str q[[V0]]
-; AS-DAG: str q[[V1]]
-; AS-DAG: str q[[V2]]
-; AS-DAG: str q[[V3]]
-; AS: ret
+
%a1 = add i64 %idx, 4
%idx2 = lshr i64 %a1, 2
@@ -172,18 +357,52 @@ entry:
; This should be lowered into LD2
define void @aarch64_ilc_idx_ld2(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx_ld2(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX2]]
+; CHECK-NEXT: [[LOAD:%.*]] = load <8 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <8 x float> [[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <8 x float> [[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_idx_ld2:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsl x8, x1, #2
+; AS-IAENABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: add x8, x8, #16
+; AS-IAENABLED-NEXT: add x10, x0, x9
+; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; AS-IAENABLED-NEXT: str q0, [x0, x9]
+; AS-IAENABLED-NEXT: str q1, [x0, x8]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_idx_ld2:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsl x8, x1, #2
+; AS-IADISABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x8, x8, #16
+; AS-IADISABLED-NEXT: add x9, x0, x9
+; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q1, q0, [x9]
+; AS-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: str q2, [x9]
+; AS-IADISABLED-NEXT: str q0, [x0, x8]
+; AS-IADISABLED-NEXT: ret
entry:
-; CHECK-LABEL: @aarch64_ilc_idx_ld2(
-; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2
-; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, ptr %ptr, i64 [[LSHR]]
-; CHECK-DAG: [[LOAD:%.+]] = load <8 x float>, ptr [[GEP]], align 16
-; CHECK: %{{.* }}= shufflevector <8 x float> [[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: %{{.* }}= shufflevector <8 x float> [[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-DAG: ret void
-
-; AS-LABEL: aarch64_ilc_idx_ld2
-; AS: ld2
-; AS: ret
+
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
@@ -203,19 +422,77 @@ entry:
; This should be lowered into LD3
define void @aarch64_ilc_idx_ld3(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx_ld3(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[A2:%.*]] = add i64 [[IDX]], 8
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A2]], 2
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX2]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX3]]
+; CHECK-NEXT: [[LOAD:%.*]] = load <12 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD3:%.*]] = load <4 x float>, ptr [[GEP3]], align 16
+; CHECK-NEXT: [[SV1:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 3, i32 6, i32 poison>
+; CHECK-NEXT: [[SV2:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 1, i32 4, i32 7, i32 poison>
+; CHECK-NEXT: [[SV3:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 2, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE:%.*]] = shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[SV1]], <4 x float> [[LD3]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE1:%.*]] = shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[SV2]], <4 x float> [[LD3]], <4 x i32> <i32 0, i32 1, i32 2, i32 6>
+; CHECK-NEXT: [[INTERLEAVED_SHUFFLE2:%.*]] = shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[M8_11:%.*]] = shufflevector <4 x float> [[SV3]], <4 x float> [[LD3]], <4 x i32> <i32 0, i32 1, i32 4, i32 7>
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE]], ptr [[GEP]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[INTERLEAVED_SHUFFLE2]], ptr [[GEP3]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_idx_ld3:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsl x8, x1, #2
+; AS-IAENABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: add x10, x0, x9
+; AS-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x10]
+; AS-IAENABLED-NEXT: add x10, x8, #16
+; AS-IAENABLED-NEXT: add x8, x8, #32
+; AS-IAENABLED-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: str q0, [x0, x9]
+; AS-IAENABLED-NEXT: str q1, [x0, x10]
+; AS-IAENABLED-NEXT: str q2, [x0, x8]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_idx_ld3:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsl x8, x1, #2
+; AS-IADISABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x10, x8, #16
+; AS-IADISABLED-NEXT: add x8, x8, #32
+; AS-IADISABLED-NEXT: add x9, x0, x9
+; AS-IADISABLED-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q0, q2, [x9]
+; AS-IADISABLED-NEXT: ldr q4, [x9, #32]
+; AS-IADISABLED-NEXT: mov v1.16b, v0.16b
+; AS-IADISABLED-NEXT: rev64 v3.4s, v2.4s
+; AS-IADISABLED-NEXT: mov v1.s[1], v0.s[3]
+; AS-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; AS-IADISABLED-NEXT: mov v1.s[2], v2.s[2]
+; AS-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; AS-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; AS-IADISABLED-NEXT: mov v2.s[2], v4.s[0]
+; AS-IADISABLED-NEXT: mov v1.s[3], v4.s[1]
+; AS-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; AS-IADISABLED-NEXT: str q1, [x9]
+; AS-IADISABLED-NEXT: str q3, [x0, x10]
+; AS-IADISABLED-NEXT: str q2, [x0, x8]
+; AS-IADISABLED-NEXT: ret
entry:
-; CHECK-LABEL: @aarch64_ilc_idx_ld3(
-; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2
-; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, ptr %ptr, i64 [[LSHR]]
-; CHECK-DAG: [[LOAD:%.+]] = load <12 x float>, ptr [[GEP]], align 16
-; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-DAG: ret void
-
-; AS-LABEL: aarch64_ilc_idx_ld3
-; AS: ld3
-; AS: ret
+
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
@@ -246,26 +523,36 @@ entry:
; This must not be lowered
define void @aarch64_ilc_i32_idx(ptr %ptr, i32 %idx) {
-; CHECK-LABEL: @aarch64_ilc_i32_idx(
-; CHECK: %idx1 = lshr i32 %idx, 2
-; CHECK-NEXT: %a1 = add i32 %idx, 4
-; CHECK-NEXT: %idx2 = lshr i32 %a1, 2
-; CHECK-NEXT: %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i32 %idx1
-; CHECK-NEXT: %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 %idx2
-; CHECK-NEXT: %ld1 = load <4 x float>, ptr %gep1, align 16
-; CHECK-NEXT: %ld2 = load <4 x float>, ptr %gep2, align 16
-; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: store <4 x float> %m0_3, ptr %gep1, align 16
-; CHECK-NEXT: store <4 x float> %m4_7, ptr %gep2, align 16
-; CHECK-NEXT: ret void
-
-; AS-LABEL: aarch64_ilc_i32_idx
-; AS-DAG: @function
-; AS-NOT: ld2
-; AS-NOT: ld3
-; AS-NOT: ld4
-; AS-DAG: ret
+; CHECK-LABEL: define void @aarch64_ilc_i32_idx(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i32 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i32 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i32 [[A1]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 [[IDX2]]
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_i32_idx:
+; AS: // %bb.0: // %entry
+; AS-NEXT: add w8, w1, #4
+; AS-NEXT: lsr w9, w1, #2
+; AS-NEXT: lsr w8, w8, #2
+; AS-NEXT: ubfiz x9, x9, #4, #32
+; AS-NEXT: ubfiz x8, x8, #4, #32
+; AS-NEXT: ldr q0, [x0, x9]
+; AS-NEXT: ldr q1, [x0, x8]
+; AS-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; AS-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; AS-NEXT: str q2, [x0, x9]
+; AS-NEXT: str q0, [x0, x8]
+; AS-NEXT: ret
entry:
%idx1 = lshr i32 %idx, 2
@@ -286,22 +573,26 @@ entry:
; Volatile loads must not be lowered
define void @aarch64_ilc_volatile(ptr %ptr) {
-; CHECK-LABEL: @aarch64_ilc_volatile(
-; CHECK: %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
-; CHECK-NEXT: %ld1 = load volatile <4 x float>, ptr %ptr, align 16
-; CHECK-NEXT: %ld2 = load <4 x float>, ptr %gep2, align 16
-; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: store <4 x float> %m0_3, ptr %ptr, align 16
-; CHECK-NEXT: store <4 x float> %m4_7, ptr %gep2, align 16
-; CHECK-NEXT: ret void
-
-; AS-LABEL: aarch64_ilc_volatile
-; AS-DAG: @function
-; AS-NOT: ld2
-; AS-NOT: ld3
-; AS-NOT: ld4
-; AS-DAG: ret
+; CHECK-LABEL: define void @aarch64_ilc_volatile(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD1:%.*]] = load volatile <4 x float>, ptr [[PTR]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_volatile:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ldr q0, [x0]
+; AS-NEXT: ldr q1, [x0, #16]
+; AS-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; AS-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; AS-NEXT: stp q2, q0, [x0]
+; AS-NEXT: ret
entry:
%gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
@@ -316,24 +607,28 @@ entry:
; This must not be lowered
define void @aarch64_ilc_depmem(ptr %ptr, i32 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_depmem(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[LD1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[M0_3:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[M4_7:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_depmem:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ldr q0, [x0]
+; AS-NEXT: uzp1 v1.4s, v0.4s, v0.4s
+; AS-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; AS-NEXT: stp q1, q0, [x0]
+; AS-NEXT: ret
entry:
-; CHECK-LABEL: @aarch64_ilc_depmem(
-; CHECK: %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
-; CHECK-NEXT: %ld1 = load <4 x float>, ptr %ptr, align 16
-; CHECK-NEXT: store <4 x float> %ld1, ptr %gep2, align 16
-; CHECK-NEXT: %ld2 = load <4 x float>, ptr %gep2, align 16
-; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: store <4 x float> %m0_3, ptr %ptr, align 16
-; CHECK-NEXT: store <4 x float> %m4_7, ptr %gep2, align 16
-; CHECK-NEXT: ret void
-
-; AS-LABEL: aarch64_ilc_depmem
-; AS-DAG: @function
-; AS-NOT: ld2
-; AS-NOT: ld3
-; AS-NOT: ld4
-; AS-DAG: ret
+
%gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
%ld1 = load <4 x float>, ptr %ptr, align 16
@@ -349,14 +644,20 @@ entry:
; This cannot be converted - insertion position cannot be determined
define void @aarch64_no_insertion_pos(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_no_insertion_pos(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i32 4
+; CHECK-NEXT: [[L0:%.*]] = load <5 x float>, ptr [[PTR]], align 32
+; CHECK-NEXT: [[L1:%.*]] = load <5 x float>, ptr [[P1]], align 32
+; CHECK-NEXT: [[S0:%.*]] = shufflevector <5 x float> [[L0]], <5 x float> [[L1]], <4 x i32> <i32 1, i32 3, i32 6, i32 8>
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <5 x float> [[L0]], <5 x float> [[L1]], <4 x i32> <i32 2, i32 4, i32 7, i32 9>
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_no_insertion_pos:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ret
entry:
-; CHECK-LABEL: @aarch64_no_insertion_pos(
-; CHECK: %p1 = getelementptr inbounds float, ptr %ptr, i32 4
-; CHECK-NEXT: %l0 = load <5 x float>, ptr %ptr
-; CHECK-NEXT: %l1 = load <5 x float>, ptr %p1
-; CHECK-NEXT: %s0 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> <i32 1, i32 3, i32 6, i32 8>
-; CHECK-NEXT: %s1 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> <i32 2, i32 4, i32 7, i32 9>
-; CHECK-NEXT: ret void
%p1 = getelementptr inbounds float, ptr %ptr, i32 4
%l0 = load <5 x float>, ptr %ptr
@@ -369,14 +670,20 @@ entry:
; This cannot be converted - the insertion position does not dominate all
; uses
define void @aarch64_insertpos_does_not_dominate(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_insertpos_does_not_dominate(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[L1:%.*]] = load <7 x float>, ptr [[P1]], align 32
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <7 x float> [[L1]], <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[L0:%.*]] = load <7 x float>, ptr [[PTR]], align 32
+; CHECK-NEXT: [[S0:%.*]] = shufflevector <7 x float> [[L0]], <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_insertpos_does_not_dominate:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ret
entry:
-; CHECK-LABEL: @aarch64_insertpos_does_not_dominate(
-; CHECK: %p1 = getelementptr inbounds float, ptr %ptr, i32 1
-; CHECK-NEXT: %l1 = load <7 x float>, ptr %p1
-; CHECK-NEXT: %s1 = shufflevector <7 x float> %l1, <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: %l0 = load <7 x float>, ptr %ptr
-; CHECK-NEXT: %s0 = shufflevector <7 x float> %l0, <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: ret void
%p1 = getelementptr inbounds float, ptr %ptr, i32 1
%l1 = load <7 x float>, ptr %p1
%s1 = shufflevector <7 x float> %l1, <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -384,3 +691,490 @@ entry:
%s0 = shufflevector <7 x float> %l0, <7 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
ret void
}
+
+define void @aarch64_ilc_const_intrinsic(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_ilc_const_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 3
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 4
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 5
+; CHECK-NEXT: [[WIDE:%.*]] = load <16 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: [[M8_11:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 2
+; CHECK-NEXT: [[M12_15:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 3
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[M8_11]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[M12_15]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_const_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: add x8, x0, #32
+; AS-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
+; AS-NEXT: stp q0, q1, [x0, #32]
+; AS-NEXT: stp q2, q3, [x0, #64]
+; AS-NEXT: ret
+entry:
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 2
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 3
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 4
+ %gep4 = getelementptr inbounds <4 x float>, ptr %ptr, i64 5
+ %wide = load <16 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+ %m12_15 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 3
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ store <4 x float> %m12_15, ptr %gep4, align 16
+ ret void
+}
+
+define void @aarch64_ilc_idx_intrinsic(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A2:%.*]] = add i64 [[IDX]], 20
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A2]], 2
+; CHECK-NEXT: [[A3:%.*]] = add i64 [[IDX]], 24
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 16
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A3]], 2
+; CHECK-NEXT: [[A4:%.*]] = add i64 [[IDX]], 28
+; CHECK-NEXT: [[IDX4:%.*]] = lshr i64 [[A4]], 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX2]]
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX4]]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX3]]
+; CHECK-NEXT: [[WIDE:%.*]] = load <16 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: [[M8_11:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 2
+; CHECK-NEXT: [[M12_15:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 3
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[M8_11]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[M12_15]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_idx_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: lsl x8, x1, #2
+; AS-NEXT: add x9, x8, #64
+; AS-NEXT: add x11, x8, #96
+; AS-NEXT: and x9, x9, #0xfffffffffffffff0
+; AS-NEXT: add x10, x0, x9
+; AS-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x10]
+; AS-NEXT: add x10, x8, #80
+; AS-NEXT: add x8, x8, #112
+; AS-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-NEXT: str q0, [x0, x9]
+; AS-NEXT: and x9, x11, #0xfffffffffffffff0
+; AS-NEXT: str q1, [x0, x10]
+; AS-NEXT: str q2, [x0, x9]
+; AS-NEXT: str q3, [x0, x8]
+; AS-NEXT: ret
+entry:
+ %a2 = add i64 %idx, 20
+ %idx2 = lshr i64 %a2, 2
+ %a3 = add i64 %idx, 24
+ %a1 = add i64 %idx, 16
+ %idx1 = lshr i64 %a1, 2
+ %idx3 = lshr i64 %a3, 2
+ %a4 = add i64 %idx, 28
+ %idx4 = lshr i64 %a4, 2
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+ %gep4 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx4
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx3
+ %wide = load <16 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+ %m12_15 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 3
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ store <4 x float> %m12_15, ptr %gep4, align 16
+ ret void
+}
+
+define void @aarch64_ilc_struct_intrinsic(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_struct_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[A2:%.*]] = add i64 [[IDX]], 8
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A2]], 2
+; CHECK-NEXT: [[A3:%.*]] = add i64 [[IDX]], 12
+; CHECK-NEXT: [[IDX4:%.*]] = lshr i64 [[A3]], 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr [[STRUCT_ILC:%.*]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX2]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX3]]
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX4]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr [[STRUCT_ILC]], ptr [[PTR]], i32 0, i32 1, i64 [[IDX1]]
+; CHECK-NEXT: [[WIDE:%.*]] = load <16 x float>, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: [[M8_11:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 2
+; CHECK-NEXT: [[M12_15:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 3
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[M8_11]], ptr [[GEP3]], align 16
+; CHECK-NEXT: store <4 x float> [[M12_15]], ptr [[GEP4]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_struct_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: lsl x8, x1, #2
+; AS-NEXT: add x9, x0, #4
+; AS-NEXT: and x10, x8, #0xfffffffffffffff0
+; AS-NEXT: add x12, x8, #32
+; AS-NEXT: add x11, x9, x10
+; AS-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x11]
+; AS-NEXT: add x11, x8, #16
+; AS-NEXT: add x8, x8, #48
+; AS-NEXT: and x11, x11, #0xfffffffffffffff0
+; AS-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-NEXT: str q0, [x9, x10]
+; AS-NEXT: and x10, x12, #0xfffffffffffffff0
+; AS-NEXT: str q1, [x9, x11]
+; AS-NEXT: str q2, [x9, x10]
+; AS-NEXT: str q3, [x9, x8]
+; AS-NEXT: ret
+entry:
+ %a1 = add i64 %idx, 4
+ %idx2 = lshr i64 %a1, 2
+ %a2 = add i64 %idx, 8
+ %idx3 = lshr i64 %a2, 2
+ %a3 = add i64 %idx, 12
+ %idx4 = lshr i64 %a3, 2
+ %gep2 = getelementptr %struct.ilc, ptr %ptr, i32 0, i32 1, i64 %idx2
+ %gep3 = getelementptr %struct.ilc, ptr %ptr, i32 0, i32 1, i64 %idx3
+ %gep4 = getelementptr %struct.ilc, ptr %ptr, i32 0, i32 1, i64 %idx4
+ %idx1 = lshr i64 %idx, 2
+ %gep1 = getelementptr %struct.ilc, ptr %ptr, i32 0, i32 1, i64 %idx1
+ %wide = load <16 x float>, ptr %gep1, align 4
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+ %m12_15 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 3
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ store <4 x float> %m12_15, ptr %gep4, align 16
+ ret void
+}
+
+define void @aarch64_ilc_idx_ld2_intrinsic(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx_ld2_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX2]]
+; CHECK-NEXT: [[WIDE:%.*]] = load <8 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_idx_ld2_intrinsic:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsl x8, x1, #2
+; AS-IAENABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: add x8, x8, #16
+; AS-IAENABLED-NEXT: add x10, x0, x9
+; AS-IAENABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; AS-IAENABLED-NEXT: str q0, [x0, x9]
+; AS-IAENABLED-NEXT: str q1, [x0, x8]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_idx_ld2_intrinsic:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsl x8, x1, #2
+; AS-IADISABLED-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: add x8, x8, #16
+; AS-IADISABLED-NEXT: add x9, x0, x9
+; AS-IADISABLED-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-IADISABLED-NEXT: ldp q1, q0, [x9]
+; AS-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: str q2, [x9]
+; AS-IADISABLED-NEXT: str q0, [x0, x8]
+; AS-IADISABLED-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %a1 = add i64 %idx, 4
+ %idx2 = lshr i64 %a1, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+ %wide = load <8 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ store <4 x float> %m0_3, ptr %gep1
+ store <4 x float> %m4_7, ptr %gep2
+ ret void
+}
+
+define void @aarch64_ilc_idx_ld3_intrinsic(ptr %ptr, i64 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_idx_ld3_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i64 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i64 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i64 [[A1]], 2
+; CHECK-NEXT: [[A2:%.*]] = add i64 [[IDX]], 8
+; CHECK-NEXT: [[IDX3:%.*]] = lshr i64 [[A2]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX2]]
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i64 [[IDX3]]
+; CHECK-NEXT: [[WIDE:%.*]] = load <12 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: [[M8_11:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[DEINTERLEAVE]], 2
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: store <4 x float> [[M8_11]], ptr [[GEP3]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_idx_ld3_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: lsl x8, x1, #2
+; AS-NEXT: and x9, x8, #0xfffffffffffffff0
+; AS-NEXT: add x10, x0, x9
+; AS-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x10]
+; AS-NEXT: add x10, x8, #16
+; AS-NEXT: add x8, x8, #32
+; AS-NEXT: and x10, x10, #0xfffffffffffffff0
+; AS-NEXT: and x8, x8, #0xfffffffffffffff0
+; AS-NEXT: str q0, [x0, x9]
+; AS-NEXT: str q1, [x0, x10]
+; AS-NEXT: str q2, [x0, x8]
+; AS-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %a1 = add i64 %idx, 4
+ %idx2 = lshr i64 %a1, 2
+ %a2 = add i64 %idx, 8
+ %idx3 = lshr i64 %a2, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx3
+ %wide = load <12 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ ret void
+}
+
+define void @aarch64_ilc_i32_idx_intrinsic(ptr %ptr, i32 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_i32_idx_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX1:%.*]] = lshr i32 [[IDX]], 2
+; CHECK-NEXT: [[A1:%.*]] = add i32 [[IDX]], 4
+; CHECK-NEXT: [[IDX2:%.*]] = lshr i32 [[A1]], 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 [[IDX1]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 [[IDX2]]
+; CHECK-NEXT: [[WIDE:%.*]] = load <8 x float>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-IAENABLED-LABEL: aarch64_ilc_i32_idx_intrinsic:
+; AS-IAENABLED: // %bb.0: // %entry
+; AS-IAENABLED-NEXT: lsr w8, w1, #2
+; AS-IAENABLED-NEXT: ubfiz x8, x8, #4, #32
+; AS-IAENABLED-NEXT: add x9, x0, x8
+; AS-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x9]
+; AS-IAENABLED-NEXT: add w9, w1, #4
+; AS-IAENABLED-NEXT: lsr w9, w9, #2
+; AS-IAENABLED-NEXT: str q0, [x0, x8]
+; AS-IAENABLED-NEXT: str q1, [x0, w9, uxtw #4]
+; AS-IAENABLED-NEXT: ret
+;
+; AS-IADISABLED-LABEL: aarch64_ilc_i32_idx_intrinsic:
+; AS-IADISABLED: // %bb.0: // %entry
+; AS-IADISABLED-NEXT: lsr w8, w1, #2
+; AS-IADISABLED-NEXT: add w9, w1, #4
+; AS-IADISABLED-NEXT: lsr w9, w9, #2
+; AS-IADISABLED-NEXT: add x8, x0, w8, uxtw #4
+; AS-IADISABLED-NEXT: ldp q1, q0, [x8]
+; AS-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; AS-IADISABLED-NEXT: str q2, [x8]
+; AS-IADISABLED-NEXT: str q0, [x0, w9, uxtw #4]
+; AS-IADISABLED-NEXT: ret
+entry:
+ %idx1 = lshr i32 %idx, 2
+ %a1 = add i32 %idx, 4
+ %idx2 = lshr i32 %a1, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i32 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 %idx2
+ %wide = load <8 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ ret void
+}
+
+define void @aarch64_ilc_volatile_intrinsic(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_ilc_volatile_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD1:%.*]] = load volatile <4 x float>, ptr [[PTR]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[WIDE:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_volatile_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ldr q0, [x0]
+; AS-NEXT: ldr q1, [x0, #16]
+; AS-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; AS-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; AS-NEXT: stp q2, q0, [x0]
+; AS-NEXT: ret
+entry:
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
+ %ld1 = load volatile <4 x float>, ptr %ptr, align 16
+ %ld2 = load <4 x float>, ptr %gep2, align 16
+ %wide = shufflevector <4 x float> %ld1, <4 x float> %ld2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ store <4 x float> %m0_3, ptr %ptr, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ ret void
+}
+
+define void @aarch64_ilc_depmem_intrinsic(ptr %ptr, i32 %idx) {
+; CHECK-LABEL: define void @aarch64_ilc_depmem_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD1:%.*]] = load <4 x float>, ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[LD1]], ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD2:%.*]] = load <4 x float>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[WIDE:%.*]] = shufflevector <4 x float> [[LD1]], <4 x float> [[LD2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE]])
+; CHECK-NEXT: [[M0_3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[M4_7:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: store <4 x float> [[M0_3]], ptr [[PTR]], align 16
+; CHECK-NEXT: store <4 x float> [[M4_7]], ptr [[GEP2]], align 16
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_ilc_depmem_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ldr q0, [x0]
+; AS-NEXT: uzp1 v1.4s, v0.4s, v0.4s
+; AS-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; AS-NEXT: stp q1, q0, [x0]
+; AS-NEXT: ret
+entry:
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i32 1
+ %ld1 = load <4 x float>, ptr %ptr, align 16
+ store <4 x float> %ld1, ptr %gep2, align 16
+ %ld2 = load <4 x float>, ptr %gep2, align 16
+ %wide = shufflevector <4 x float> %ld1, <4 x float> %ld2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide)
+ %m0_3 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ store <4 x float> %m0_3, ptr %ptr, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ ret void
+}
+
+define void @aarch64_no_insertion_pos_intrinsic(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_no_insertion_pos_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i32 4
+; CHECK-NEXT: [[L0:%.*]] = load <5 x float>, ptr [[PTR]], align 32
+; CHECK-NEXT: [[L1:%.*]] = load <5 x float>, ptr [[P1]], align 32
+; CHECK-NEXT: [[WIDE:%.*]] = shufflevector <5 x float> [[L0]], <5 x float> [[L1]], <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9>
+; CHECK-NEXT: [[DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[WIDE]])
+; CHECK-NEXT: [[S0:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT: [[S1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_no_insertion_pos_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ret
+entry:
+ %p1 = getelementptr inbounds float, ptr %ptr, i32 4
+ %l0 = load <5 x float>, ptr %ptr
+ %l1 = load <5 x float>, ptr %p1
+ %wide = shufflevector <5 x float> %l0, <5 x float> %l1, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9>
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide)
+ %s0 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %s1 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ ret void
+}
+
+define void @aarch64_insertpos_does_not_dominate_intrinsic(ptr %ptr) {
+; CHECK-LABEL: define void @aarch64_insertpos_does_not_dominate_intrinsic(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[L1:%.*]] = load <7 x float>, ptr [[P1]], align 32
+; CHECK-NEXT: [[L1_WIDE:%.*]] = shufflevector <7 x float> [[L1]], <7 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[L1_DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[L1_WIDE]])
+; CHECK-NEXT: [[S1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[L1_DEINTERLEAVE]], 0
+; CHECK-NEXT: [[L0:%.*]] = load <7 x float>, ptr [[PTR]], align 32
+; CHECK-NEXT: [[L0_WIDE:%.*]] = shufflevector <7 x float> [[L0]], <7 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[L0_DEINTERLEAVE:%.*]] = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[L0_WIDE]])
+; CHECK-NEXT: [[S0:%.*]] = extractvalue { <4 x float>, <4 x float> } [[L0_DEINTERLEAVE]], 0
+; CHECK-NEXT: ret void
+;
+; AS-LABEL: aarch64_insertpos_does_not_dominate_intrinsic:
+; AS: // %bb.0: // %entry
+; AS-NEXT: ret
+entry:
+ %p1 = getelementptr inbounds float, ptr %ptr, i32 1
+ %l1 = load <7 x float>, ptr %p1
+ %l1.wide = shufflevector <7 x float> %l1, <7 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+ %l1.deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %l1.wide)
+ %s1 = extractvalue { <4 x float>, <4 x float> } %l1.deinterleave, 0
+ %l0 = load <7 x float>, ptr %ptr
+ %l0.wide = shufflevector <7 x float> %l0, <7 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+ %l0.deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %l0.wide)
+ %s0 = extractvalue { <4 x float>, <4 x float> } %l0.deinterleave, 0
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
index 8cb1ac4ae548a..434fb08c3d4b6 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s
+; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s --mattr=+complxnum,+neon -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target triple = "aarch64"
; Expected to transform
@@ -92,33 +93,62 @@ entry:
; *p2 = (a * b) * c;
; return d * c;
define <4 x float> @multiple_muls_shuffle_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
-; CHECK-LABEL: multiple_muls_shuffle_external:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov d5, v0.d[1]
-; CHECK-NEXT: mov d6, v1.d[1]
-; CHECK-NEXT: mov d4, v2.d[1]
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-NEXT: zip1 v16.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v16.2s, v7.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v7.2s
-; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v6.2s
-; CHECK-NEXT: zip1 v6.2s, v2.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v16.2s
-; CHECK-NEXT: fmul v17.2s, v6.2s, v5.2s
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: fmul v5.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmla v17.2s, v1.2s, v4.2s
-; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-NEXT: str d1, [x0]
-; CHECK-NEXT: fneg v16.2s, v5.2s
-; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-NEXT: fmla v16.2s, v1.2s, v6.2s
-; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d5, v0.d[1]
+; CHECK-IAENABLED-NEXT: mov d6, v1.d[1]
+; CHECK-IAENABLED-NEXT: mov d4, v2.d[1]
+; CHECK-IAENABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v16.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip2 v1.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v16.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fneg v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip1 v6.2s, v2.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v4.2s, v2.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmla v1.2s, v0.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmul v17.2s, v6.2s, v5.2s
+; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v4.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmla v17.2s, v1.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
+; CHECK-IAENABLED-NEXT: str d1, [x0]
+; CHECK-IAENABLED-NEXT: fneg v16.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
+; CHECK-IAENABLED-NEXT: fmla v16.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov d5, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d6, v1.d[1]
+; CHECK-IADISABLED-NEXT: mov d4, v2.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v16.2s, v1.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip1 v5.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v1.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip2 v6.2s, v2.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v16.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v5.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmul v0.2s, v0.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fneg v7.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmul v17.2s, v6.2s, v1.2s
+; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmla v7.2s, v5.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fneg v5.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v7.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v7.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
+; CHECK-IADISABLED-NEXT: str d7, [x0]
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v5.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q1, [x1]
+; CHECK-IADISABLED-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
%strided.vec88 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
@@ -157,31 +187,64 @@ entry:
; Expected to transform partially (only d * c).
; Shows that ld2 is not generated for `c` although it used by both complex `d * c` and non-complex `(a * b) * c` instruction chains.
define <4 x float> @multiple_muls_shuffle_external_with_loads(ptr %ptr_a, ptr %ptr_b, ptr %ptr_c, ptr %ptr_d, ptr %p1, ptr %p2) {
-; CHECK-LABEL: multiple_muls_shuffle_external_with_loads:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-NEXT: ld2 { v2.2s, v3.2s }, [x1]
-; CHECK-NEXT: fmul v4.2s, v3.2s, v1.2s
-; CHECK-NEXT: fmul v6.2s, v2.2s, v1.2s
-; CHECK-NEXT: fneg v4.2s, v4.2s
-; CHECK-NEXT: fmla v6.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: str d4, [x4]
-; CHECK-NEXT: ldr q5, [x2]
-; CHECK-NEXT: mov d7, v5.d[1]
-; CHECK-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v6.2s
-; CHECK-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-NEXT: fneg v2.2s, v6.2s
-; CHECK-NEXT: fmla v2.2s, v4.2s, v0.2s
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: st2 { v2.2s, v3.2s }, [x5]
-; CHECK-NEXT: ldr q1, [x3]
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_with_loads:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v2.2s, v3.2s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v4.2s, v3.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v2.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fneg v4.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-IAENABLED-NEXT: str d4, [x4]
+; CHECK-IAENABLED-NEXT: ldr q5, [x2]
+; CHECK-IAENABLED-NEXT: mov d7, v5.d[1]
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: zip2 v1.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v3.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v3.2s, v4.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x5]
+; CHECK-IAENABLED-NEXT: ldr q1, [x3]
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_with_loads:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-IADISABLED-NEXT: ldr q1, [x1]
+; CHECK-IADISABLED-NEXT: mov d2, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d3, v1.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: zip2 v5.2s, v1.2s, v3.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v3.2s
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmla v2.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v4.2s
+; CHECK-IADISABLED-NEXT: str d2, [x4]
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: ldr q3, [x2]
+; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fneg v5.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v2.2s, v0.2s
+; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v2.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v5.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q1, [x5]
+; CHECK-IADISABLED-NEXT: ldr q1, [x3]
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #0
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #90
+; CHECK-IADISABLED-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -225,39 +288,74 @@ entry:
; *p2 = (a * b) * (d * c);
; return d * c;
define <4 x float> @multiple_muls_mul_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
-; CHECK-LABEL: multiple_muls_mul_external:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov d4, v0.d[1]
-; CHECK-NEXT: mov d5, v1.d[1]
-; CHECK-NEXT: mov d16, v2.d[1]
-; CHECK-NEXT: mov d17, v3.d[1]
-; CHECK-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v19.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v16.2s, v3.2s, v17.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-NEXT: fmul v5.2s, v19.2s, v16.2s
-; CHECK-NEXT: fmul v16.2s, v2.2s, v16.2s
-; CHECK-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-NEXT: fneg v4.2s, v18.2s
-; CHECK-NEXT: fmla v5.2s, v3.2s, v2.2s
-; CHECK-NEXT: fneg v2.2s, v16.2s
-; CHECK-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-NEXT: fmla v2.2s, v3.2s, v19.2s
-; CHECK-NEXT: fmul v0.2s, v7.2s, v5.2s
-; CHECK-NEXT: fmul v17.2s, v4.2s, v5.2s
-; CHECK-NEXT: str d4, [x0]
-; CHECK-NEXT: fmla v17.2s, v2.2s, v7.2s
-; CHECK-NEXT: fneg v16.2s, v0.2s
-; CHECK-NEXT: zip1 v0.4s, v2.4s, v5.4s
-; CHECK-NEXT: fmla v16.2s, v2.2s, v4.2s
-; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: multiple_muls_mul_external:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d4, v0.d[1]
+; CHECK-IAENABLED-NEXT: mov d5, v1.d[1]
+; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d17, v3.d[1]
+; CHECK-IAENABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v7.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v19.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: zip2 v2.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: zip2 v16.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: fmul v18.2s, v6.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v19.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmul v16.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmul v7.2s, v0.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fneg v4.2s, v18.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v2.2s
+; CHECK-IAENABLED-NEXT: fneg v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmla v7.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v1.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v3.2s, v19.2s
+; CHECK-IAENABLED-NEXT: fmul v0.2s, v7.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmul v17.2s, v4.2s, v5.2s
+; CHECK-IAENABLED-NEXT: str d4, [x0]
+; CHECK-IAENABLED-NEXT: fmla v17.2s, v2.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fneg v16.2s, v0.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.4s, v2.4s, v5.4s
+; CHECK-IAENABLED-NEXT: fmla v16.2s, v2.2s, v4.2s
+; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_mul_external:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
+; CHECK-IADISABLED-NEXT: mov d6, v2.d[1]
+; CHECK-IADISABLED-NEXT: mov d7, v3.d[1]
+; CHECK-IADISABLED-NEXT: zip1 v16.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip2 v17.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip2 v18.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v16.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v0.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmul v7.2s, v4.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmul v0.2s, v2.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmla v7.2s, v3.2s, v2.2s
+; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fneg v0.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmla v2.2s, v1.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v5.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v0.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v3.2s, v2.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
+; CHECK-IADISABLED-NEXT: str d2, [x0]
+; CHECK-IADISABLED-NEXT: fmla v3.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v7.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: str q1, [x1]
+; CHECK-IADISABLED-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
%strided.vec126 = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 1, i32 3>
@@ -381,3 +479,502 @@ entry:
store <4 x double> %interleaved.vec145, ptr %p2, align 8
ret void
}
+
+
+define <4 x float> @mul_triangle_intrinsic(<4 x float> %a, <4 x float> %b, ptr %p) {
+; CHECK-LABEL: mul_triangle_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: fcmla v3.4s, v0.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v3.4s, v0.4s, v1.4s, #90
+; CHECK-NEXT: fcmla v2.4s, v3.4s, v0.4s, #0
+; CHECK-NEXT: str q3, [x0]
+; CHECK-NEXT: fcmla v2.4s, v3.4s, v0.4s, #90
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+entry:
+ %a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
+ %strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
+ %strided.vec35 = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 1
+ %b.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %b)
+ %strided.vec37 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 0
+ %strided.vec38 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 1
+ %0 = fmul fast <2 x float> %strided.vec37, %strided.vec
+ %1 = fmul fast <2 x float> %strided.vec38, %strided.vec35
+ %2 = fsub fast <2 x float> %0, %1
+ %3 = fmul fast <2 x float> %2, %strided.vec35
+ %4 = fmul fast <2 x float> %strided.vec38, %strided.vec
+ %5 = fmul fast <2 x float> %strided.vec35, %strided.vec37
+ %6 = fadd fast <2 x float> %4, %5
+ %otheruse = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %2, <2 x float> %6)
+ store <4 x float> %otheruse, ptr %p
+ %7 = fmul fast <2 x float> %6, %strided.vec
+ %8 = fadd fast <2 x float> %3, %7
+ %9 = fmul fast <2 x float> %2, %strided.vec
+ %10 = fmul fast <2 x float> %6, %strided.vec35
+ %11 = fsub fast <2 x float> %9, %10
+ %interleaved.vec = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %11, <2 x float> %8)
+ ret <4 x float> %interleaved.vec
+}
+
+
+define <4 x float> @mul_triangle_external_use_intrinsic(<4 x float> %a, <4 x float> %b, ptr %p) {
+; CHECK-LABEL: mul_triangle_external_use_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d2, v0.d[1]
+; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
+; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
+; CHECK-NEXT: str d2, [x0]
+; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
+; CHECK-NEXT: fneg v3.2s, v3.2s
+; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
+; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
+; CHECK-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-NEXT: rev64 v0.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.4s, v3.4s, v0.4s
+; CHECK-NEXT: ret
+entry:
+ %a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
+ %strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
+ %strided.vec35 = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 1
+ %b.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %b)
+ %strided.vec37 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 0
+ %strided.vec38 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 1
+ %0 = fmul fast <2 x float> %strided.vec37, %strided.vec
+ %1 = fmul fast <2 x float> %strided.vec38, %strided.vec35
+ %2 = fsub fast <2 x float> %0, %1
+ %3 = fmul fast <2 x float> %2, %strided.vec35
+ %4 = fmul fast <2 x float> %strided.vec38, %strided.vec
+ %5 = fmul fast <2 x float> %strided.vec35, %strided.vec37
+ %6 = fadd fast <2 x float> %4, %5
+ store <2 x float> %6, ptr %p
+ %7 = fmul fast <2 x float> %6, %strided.vec
+ %8 = fadd fast <2 x float> %3, %7
+ %9 = fmul fast <2 x float> %2, %strided.vec
+ %10 = fmul fast <2 x float> %6, %strided.vec35
+ %11 = fsub fast <2 x float> %9, %10
+ %interleaved.vec = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %11, <2 x float> %8)
+ ret <4 x float> %interleaved.vec
+}
+
+
+define <4 x float> @multiple_muls_shuffle_external_intrinsic(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
+; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d5, v0.d[1]
+; CHECK-IAENABLED-NEXT: mov d6, v1.d[1]
+; CHECK-IAENABLED-NEXT: mov d4, v2.d[1]
+; CHECK-IAENABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip2 v16.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v1.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v16.2s, v7.2s
+; CHECK-IAENABLED-NEXT: zip1 v7.2s, v2.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v4.2s, v2.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fneg v6.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fmul v17.2s, v7.2s, v5.2s
+; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IAENABLED-NEXT: fmul v1.2s, v4.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmla v17.2s, v6.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
+; CHECK-IAENABLED-NEXT: str d6, [x0]
+; CHECK-IAENABLED-NEXT: fneg v16.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
+; CHECK-IAENABLED-NEXT: fmla v16.2s, v6.2s, v7.2s
+; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov d4, v1.d[1]
+; CHECK-IADISABLED-NEXT: mov d5, v0.d[1]
+; CHECK-IADISABLED-NEXT: zip1 v6.2s, v1.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-IADISABLED-NEXT: mov d5, v2.d[1]
+; CHECK-IADISABLED-NEXT: fmul v4.2s, v6.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v7.2s
+; CHECK-IADISABLED-NEXT: zip2 v7.2s, v2.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v5.2s, v2.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v16.2s, v7.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v4.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fneg v6.2s, v16.2s
+; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v1.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
+; CHECK-IADISABLED-NEXT: str d1, [x0]
+; CHECK-IADISABLED-NEXT: mov v6.d[1], v4.d[0]
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
+; CHECK-IADISABLED-NEXT: rev64 v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v6.4s, v4.4s
+; CHECK-IADISABLED-NEXT: str q2, [x1]
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
+ %strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
+ %strided.vec88 = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 1
+ %b.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %b)
+ %strided.vec90 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 0
+ %strided.vec91 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 1
+ %0 = fmul fast <2 x float> %strided.vec91, %strided.vec
+ %1 = fmul fast <2 x float> %strided.vec90, %strided.vec88
+ %2 = fadd fast <2 x float> %0, %1
+ %3 = fmul fast <2 x float> %strided.vec90, %strided.vec
+ %4 = fmul fast <2 x float> %strided.vec91, %strided.vec88
+ %5 = fsub fast <2 x float> %3, %4
+ store <2 x float> %5, ptr %p1
+ %c.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %c)
+ %strided.vec93 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 0
+ %strided.vec94 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 1
+ %6 = fmul fast <2 x float> %strided.vec94, %5
+ %7 = fmul fast <2 x float> %strided.vec93, %2
+ %8 = fadd fast <2 x float> %6, %7
+ %9 = fmul fast <2 x float> %strided.vec93, %5
+ %10 = fmul fast <2 x float> %strided.vec94, %2
+ %11 = fsub fast <2 x float> %9, %10
+ %interleaved.vec = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %11, <2 x float> %8)
+ store <4 x float> %interleaved.vec, ptr %p2
+ %d.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %d)
+ %strided.vec96 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 0
+ %strided.vec97 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 1
+ %12 = fmul fast <2 x float> %strided.vec96, %strided.vec94
+ %13 = fmul fast <2 x float> %strided.vec97, %strided.vec93
+ %14 = fadd fast <2 x float> %13, %12
+ %15 = fmul fast <2 x float> %strided.vec96, %strided.vec93
+ %16 = fmul fast <2 x float> %strided.vec97, %strided.vec94
+ %17 = fsub fast <2 x float> %15, %16
+ %interleaved.vec98 = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %17, <2 x float> %14)
+ ret <4 x float> %interleaved.vec98
+}
+
+
+define <4 x float> @multiple_muls_shuffle_external_with_loads_intrinsic(ptr %ptr_a, ptr %ptr_b, ptr %ptr_c, ptr %ptr_d, ptr %p1, ptr %p2) {
+; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v2.2s, v3.2s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v4.2s, v3.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v2.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fneg v4.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-IAENABLED-NEXT: str d4, [x4]
+; CHECK-IAENABLED-NEXT: ldr q5, [x2]
+; CHECK-IAENABLED-NEXT: mov d7, v5.d[1]
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: zip2 v1.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v3.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v3.2s, v4.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x5]
+; CHECK-IAENABLED-NEXT: ldr q1, [x3]
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
+; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-IADISABLED-NEXT: ldr q1, [x1]
+; CHECK-IADISABLED-NEXT: mov d2, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d3, v1.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: zip2 v5.2s, v1.2s, v3.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v3.2s
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmla v2.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v4.2s
+; CHECK-IADISABLED-NEXT: str d2, [x4]
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: ldr q3, [x2]
+; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fneg v5.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v2.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v2.2s, v4.2s
+; CHECK-IADISABLED-NEXT: mov v5.d[1], v1.d[0]
+; CHECK-IADISABLED-NEXT: rev64 v0.4s, v5.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
+; CHECK-IADISABLED-NEXT: str q1, [x5]
+; CHECK-IADISABLED-NEXT: ldr q1, [x3]
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #0
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #90
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %a = load <4 x float>, ptr %ptr_a
+ %a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
+ %strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
+ %strided.vec88 = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 1
+ %b = load <4 x float>, ptr %ptr_b
+ %b.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %b)
+ %strided.vec90 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 0
+ %strided.vec91 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 1
+ %0 = fmul fast <2 x float> %strided.vec91, %strided.vec
+ %1 = fmul fast <2 x float> %strided.vec90, %strided.vec88
+ %2 = fadd fast <2 x float> %0, %1
+ %3 = fmul fast <2 x float> %strided.vec90, %strided.vec
+ %4 = fmul fast <2 x float> %strided.vec91, %strided.vec88
+ %5 = fsub fast <2 x float> %3, %4
+ store <2 x float> %5, ptr %p1
+ %c = load <4 x float>, ptr %ptr_c
+ %c.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %c)
+ %strided.vec93 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 0
+ %strided.vec94 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 1
+ %6 = fmul fast <2 x float> %strided.vec94, %5
+ %7 = fmul fast <2 x float> %strided.vec93, %2
+ %8 = fadd fast <2 x float> %6, %7
+ %9 = fmul fast <2 x float> %strided.vec93, %5
+ %10 = fmul fast <2 x float> %strided.vec94, %2
+ %11 = fsub fast <2 x float> %9, %10
+ %interleaved.vec = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %11, <2 x float> %8)
+ store <4 x float> %interleaved.vec, ptr %p2
+ %d = load <4 x float>, ptr %ptr_d
+ %d.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %d)
+ %strided.vec96 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 0
+ %strided.vec97 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 1
+ %12 = fmul fast <2 x float> %strided.vec96, %strided.vec94
+ %13 = fmul fast <2 x float> %strided.vec97, %strided.vec93
+ %14 = fadd fast <2 x float> %13, %12
+ %15 = fmul fast <2 x float> %strided.vec96, %strided.vec93
+ %16 = fmul fast <2 x float> %strided.vec97, %strided.vec94
+ %17 = fsub fast <2 x float> %15, %16
+ %interleaved.vec98 = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %17, <2 x float> %14)
+ ret <4 x float> %interleaved.vec98
+}
+
+
+define <4 x float> @multiple_muls_mul_external_intrinsic(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
+; CHECK-IAENABLED-LABEL: multiple_muls_mul_external_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d4, v0.d[1]
+; CHECK-IAENABLED-NEXT: mov d5, v1.d[1]
+; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d17, v3.d[1]
+; CHECK-IAENABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v7.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip2 v19.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: zip1 v2.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: zip2 v16.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: fmul v18.2s, v6.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmul v16.2s, v19.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmul v7.2s, v0.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fneg v4.2s, v18.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v19.2s
+; CHECK-IAENABLED-NEXT: fmla v7.2s, v1.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v1.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fneg v0.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmla v0.2s, v3.2s, v2.2s
+; CHECK-IAENABLED-NEXT: fmul v2.2s, v4.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmul v3.2s, v7.2s, v5.2s
+; CHECK-IAENABLED-NEXT: str d4, [x0]
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v0.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fneg v1.2s, v3.2s
+; CHECK-IAENABLED-NEXT: fmla v1.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: mov v0.d[1], v5.d[0]
+; CHECK-IAENABLED-NEXT: rev64 v3.4s, v0.4s
+; CHECK-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x1]
+; CHECK-IAENABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: multiple_muls_mul_external_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
+; CHECK-IADISABLED-NEXT: mov d6, v2.d[1]
+; CHECK-IADISABLED-NEXT: mov d7, v3.d[1]
+; CHECK-IADISABLED-NEXT: zip1 v16.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip2 v17.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip2 v18.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v16.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v0.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmul v7.2s, v4.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmul v0.2s, v2.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmla v7.2s, v3.2s, v2.2s
+; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fneg v0.2s, v0.2s
+; CHECK-IADISABLED-NEXT: fmla v2.2s, v1.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v5.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v0.2s, v3.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmul v3.2s, v2.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
+; CHECK-IADISABLED-NEXT: str d2, [x0]
+; CHECK-IADISABLED-NEXT: fmla v3.2s, v0.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v2.2s
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v7.d[0]
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v3.d[0]
+; CHECK-IADISABLED-NEXT: rev64 v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: rev64 v4.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v4.4s
+; CHECK-IADISABLED-NEXT: str q1, [x1]
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
+ %strided.vec = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 0
+ %strided.vec126 = extractvalue { <2 x float>, <2 x float> } %a.deinterleaved, 1
+ %b.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %b)
+ %strided.vec128 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 0
+ %strided.vec129 = extractvalue { <2 x float>, <2 x float> } %b.deinterleaved, 1
+ %0 = fmul nnan ninf contract <2 x float> %strided.vec, %strided.vec129
+ %1 = fmul nnan ninf contract <2 x float> %strided.vec126, %strided.vec128
+ %2 = fadd nnan ninf contract <2 x float> %1, %0
+ %3 = fmul nnan ninf contract <2 x float> %strided.vec, %strided.vec128
+ %4 = fmul nnan ninf contract <2 x float> %strided.vec126, %strided.vec129
+ %5 = fsub nnan ninf contract <2 x float> %3, %4
+ store <2 x float> %5, ptr %p1
+ %c.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %c)
+ %strided.vec131 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 0
+ %strided.vec132 = extractvalue { <2 x float>, <2 x float> } %c.deinterleaved, 1
+ %d.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %d)
+ %strided.vec134 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 0
+ %strided.vec135 = extractvalue { <2 x float>, <2 x float> } %d.deinterleaved, 1
+ %6 = fmul nnan ninf contract <2 x float> %strided.vec131, %strided.vec135
+ %7 = fmul nnan ninf contract <2 x float> %strided.vec132, %strided.vec134
+ %8 = fadd nnan ninf contract <2 x float> %7, %6
+ %9 = fmul nnan ninf contract <2 x float> %strided.vec131, %strided.vec134
+ %10 = fmul nnan ninf contract <2 x float> %strided.vec132, %strided.vec135
+ %11 = fsub nnan ninf contract <2 x float> %9, %10
+ %12 = fmul nnan ninf contract <2 x float> %5, %8
+ %13 = fmul nnan ninf contract <2 x float> %2, %11
+ %14 = fadd nnan ninf contract <2 x float> %13, %12
+ %15 = fmul nnan ninf contract <2 x float> %5, %11
+ %16 = fmul nnan ninf contract <2 x float> %2, %8
+ %17 = fsub nnan ninf contract <2 x float> %15, %16
+ %interleaved.vec = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %17, <2 x float> %14)
+ store <4 x float> %interleaved.vec, ptr %p2
+ %interleaved.vec136 = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %11, <2 x float> %8)
+ ret <4 x float> %interleaved.vec136
+}
+
+
+define void @mul_add_common_mul_add_mul_intrinsic(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d, <4 x double> %e, <4 x double> %f, <4 x double> %g, <4 x double> %h, ptr %p1, ptr %p2) {
+; CHECK-LABEL: mul_add_common_mul_add_mul_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v16.2d, #0000000000000000
+; CHECK-NEXT: movi v17.2d, #0000000000000000
+; CHECK-NEXT: ldr q19, [sp, #112]
+; CHECK-NEXT: ldp q18, q20, [sp, #80]
+; CHECK-NEXT: ldr q21, [sp, #64]
+; CHECK-NEXT: movi v22.2d, #0000000000000000
+; CHECK-NEXT: fcmla v16.2d, v18.2d, v19.2d, #0
+; CHECK-NEXT: fcmla v17.2d, v21.2d, v20.2d, #0
+; CHECK-NEXT: fcmla v22.2d, v1.2d, v3.2d, #0
+; CHECK-NEXT: fcmla v16.2d, v18.2d, v19.2d, #90
+; CHECK-NEXT: movi v18.2d, #0000000000000000
+; CHECK-NEXT: fcmla v17.2d, v21.2d, v20.2d, #90
+; CHECK-NEXT: fcmla v22.2d, v1.2d, v3.2d, #90
+; CHECK-NEXT: fcmla v16.2d, v5.2d, v7.2d, #0
+; CHECK-NEXT: fcmla v18.2d, v0.2d, v2.2d, #0
+; CHECK-NEXT: fcmla v17.2d, v4.2d, v6.2d, #0
+; CHECK-NEXT: fcmla v16.2d, v5.2d, v7.2d, #90
+; CHECK-NEXT: fcmla v18.2d, v0.2d, v2.2d, #90
+; CHECK-NEXT: fcmla v17.2d, v4.2d, v6.2d, #90
+; CHECK-NEXT: ldp q3, q0, [sp, #32]
+; CHECK-NEXT: ldp q2, q1, [sp]
+; CHECK-NEXT: fsub v4.2d, v22.2d, v16.2d
+; CHECK-NEXT: fsub v5.2d, v18.2d, v17.2d
+; CHECK-NEXT: fcmla v16.2d, v0.2d, v1.2d, #0
+; CHECK-NEXT: fcmla v17.2d, v3.2d, v2.2d, #0
+; CHECK-NEXT: stp q5, q4, [x0]
+; CHECK-NEXT: fcmla v16.2d, v0.2d, v1.2d, #90
+; CHECK-NEXT: fcmla v17.2d, v3.2d, v2.2d, #90
+; CHECK-NEXT: stp q17, q16, [x1]
+; CHECK-NEXT: ret
+entry:
+ %a.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %a)
+ %strided.vec = extractvalue { <2 x double>, <2 x double> } %a.deinterleaved, 0
+ %strided.vec123 = extractvalue { <2 x double>, <2 x double> } %a.deinterleaved, 1
+ %b.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %b)
+ %strided.vec125 = extractvalue { <2 x double>, <2 x double> } %b.deinterleaved, 0
+ %strided.vec126 = extractvalue { <2 x double>, <2 x double> } %b.deinterleaved, 1
+ %0 = fmul fast <2 x double> %strided.vec125, %strided.vec
+ %1 = fmul fast <2 x double> %strided.vec126, %strided.vec
+ %2 = fmul fast <2 x double> %strided.vec125, %strided.vec123
+ %3 = fadd fast <2 x double> %1, %2
+ %c.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %c)
+ %strided.vec128 = extractvalue { <2 x double>, <2 x double> } %c.deinterleaved, 0
+ %strided.vec129 = extractvalue { <2 x double>, <2 x double> } %c.deinterleaved, 1
+ %d.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %d)
+ %strided.vec131 = extractvalue { <2 x double>, <2 x double> } %d.deinterleaved, 0
+ %strided.vec132 = extractvalue { <2 x double>, <2 x double> } %d.deinterleaved, 1
+ %4 = fmul fast <2 x double> %strided.vec131, %strided.vec128
+ %5 = fmul fast <2 x double> %strided.vec132, %strided.vec129
+ %6 = fmul fast <2 x double> %strided.vec132, %strided.vec128
+ %7 = fmul fast <2 x double> %strided.vec131, %strided.vec129
+ %8 = fsub fast <2 x double> %4, %5
+ %g.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %g)
+ %strided.vec134 = extractvalue { <2 x double>, <2 x double> } %g.deinterleaved, 0
+ %strided.vec135 = extractvalue { <2 x double>, <2 x double> } %g.deinterleaved, 1
+ %h.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %h)
+ %strided.vec137 = extractvalue { <2 x double>, <2 x double> } %h.deinterleaved, 0
+ %strided.vec138 = extractvalue { <2 x double>, <2 x double> } %h.deinterleaved, 1
+ %9 = fmul fast <2 x double> %strided.vec138, %strided.vec134
+ %10 = fmul fast <2 x double> %strided.vec137, %strided.vec135
+ %11 = fmul fast <2 x double> %strided.vec137, %strided.vec134
+ %12 = fmul fast <2 x double> %strided.vec135, %strided.vec138
+ %13 = fsub fast <2 x double> %11, %12
+ %14 = fadd fast <2 x double> %13, %8
+ %15 = fadd fast <2 x double> %6, %7
+ %16 = fadd fast <2 x double> %15, %9
+ %17 = fadd fast <2 x double> %16, %10
+ %18 = fmul fast <2 x double> %strided.vec126, %strided.vec123
+ %19 = fadd fast <2 x double> %18, %14
+ %20 = fsub fast <2 x double> %0, %19
+ %21 = fsub fast <2 x double> %3, %17
+ %interleaved.vec = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %20, <2 x double> %21)
+ store <4 x double> %interleaved.vec, ptr %p1, align 8
+ %e.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %e)
+ %strided.vec140 = extractvalue { <2 x double>, <2 x double> } %e.deinterleaved, 0
+ %strided.vec141 = extractvalue { <2 x double>, <2 x double> } %e.deinterleaved, 1
+ %f.deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %f)
+ %strided.vec143 = extractvalue { <2 x double>, <2 x double> } %f.deinterleaved, 0
+ %strided.vec144 = extractvalue { <2 x double>, <2 x double> } %f.deinterleaved, 1
+ %22 = fmul fast <2 x double> %strided.vec143, %strided.vec140
+ %23 = fmul fast <2 x double> %strided.vec144, %strided.vec140
+ %24 = fmul fast <2 x double> %strided.vec143, %strided.vec141
+ %25 = fadd fast <2 x double> %22, %14
+ %26 = fmul fast <2 x double> %strided.vec144, %strided.vec141
+ %27 = fsub fast <2 x double> %25, %26
+ %28 = fadd fast <2 x double> %24, %17
+ %29 = fadd fast <2 x double> %28, %23
+ %interleaved.vec145 = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %27, <2 x double> %29)
+ store <4 x double> %interleaved.vec145, ptr %p2, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll b/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
index bda66d43e4b32..951f356ee1da5 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-load-interleaved.ll
@@ -1,23 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE
-; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE
+; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IADISABLED
+; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IADISABLED
; Note: These tests show that LDNP is not (yet) prioritized for interleaved loads.
define void @test_ldnp_interleaved_load2_v2i32(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2i32:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-NEXT: str d0, [x1]
-; CHECK-LE-NEXT: str d1, [x2]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2i32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
-; CHECK-BE-NEXT: st1 { v1.2s }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <4 x i32>, ptr %ptr, align 4, !nontemporal !0
%v0 = shufflevector <4 x i32> %loaded, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
@@ -28,19 +50,37 @@ entry:
}
define void @test_ldnp_interleaved_load2_v4i16(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4i16:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-NEXT: str d0, [x1]
-; CHECK-LE-NEXT: str d1, [x2]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4i16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
-; CHECK-BE-NEXT: st1 { v1.4h }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: xtn v1.4h, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <8 x i16>, ptr %ptr, align 2, !nontemporal !0
%v0 = shufflevector <8 x i16> %loaded, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -51,19 +91,37 @@ entry:
}
define void @test_ldnp_interleaved_load2_v8i8(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8i8:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.8b, v1.8b }, [x0]
-; CHECK-LE-NEXT: str d0, [x1]
-; CHECK-LE-NEXT: str d1, [x2]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8i8:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.8b, v1.8b }, [x0]
-; CHECK-BE-NEXT: st1 { v0.8b }, [x1]
-; CHECK-BE-NEXT: st1 { v1.8b }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: xtn v1.8b, v0.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.16b, v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <16 x i8>, ptr %ptr, align 1, !nontemporal !0
%v0 = shufflevector <16 x i8> %loaded, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -74,19 +132,39 @@ entry:
}
define void @test_ldnp_interleaved_load2_v2f32(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2f32:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-NEXT: str d0, [x1]
-; CHECK-LE-NEXT: str d1, [x2]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2f32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.2s, v1.2s }, [x0]
-; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
-; CHECK-BE-NEXT: st1 { v1.2s }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <4 x float>, ptr %ptr, align 4, !nontemporal !0
%v0 = shufflevector <4 x float> %loaded, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -97,195 +175,2791 @@ entry:
}
define void @test_ldnp_interleaved_load2_v4f16(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4f16:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-LE-NEXT: str d0, [x1]
-; CHECK-LE-NEXT: str d1, [x2]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <8 x half> %loaded, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %v1 = shufflevector <8 x half> %loaded, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ store <4 x half> %v0, ptr %out0
+ store <4 x half> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v2i64(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <4 x i64>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <4 x i64> %loaded, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+ %v1 = shufflevector <4 x i64> %loaded, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+ store <2 x i64> %v0, ptr %out0
+ store <2 x i64> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v4i32(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %v1 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ store <4 x i32> %v0, ptr %out0
+ store <4 x i32> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v8i16(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %v1 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ store <8 x i16> %v0, ptr %out0
+ store <8 x i16> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v16i8(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.16b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.16b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <32 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %v0 = shufflevector <32 x i8> %loaded, <32 x i8> poison,
+ <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14,
+ i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+ %v1 = shufflevector <32 x i8> %loaded, <32 x i8> poison,
+ <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15,
+ i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+ store <16 x i8> %v0, ptr %out0
+ store <16 x i8> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v2f64(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <4 x double>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <4 x double> %loaded, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+ %v1 = shufflevector <4 x double> %loaded, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+ store <2 x double> %v0, ptr %out0
+ store <2 x double> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v4f32(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x float>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <8 x float> %loaded, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %v1 = shufflevector <8 x float> %loaded, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ store <4 x float> %v0, ptr %out0
+ store <4 x float> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v8f16(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <16 x half> %loaded, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %v1 = shufflevector <16 x half> %loaded, <16 x half> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ store <8 x half> %v0, ptr %out0
+ store <8 x half> %v1, ptr %out1
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v2i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v3.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-LE-IADISABLED-NEXT: ext v1.8b, v0.8b, v2.8b, #4
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d2, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ldr d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov d2, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v4.s[1], v2.s[1]
+; CHECK-BE-IADISABLED-NEXT: ext v2.8b, v0.8b, v3.8b, #4
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <6 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 0, i32 3>
+ %v1 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 1, i32 4>
+ %v2 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 2, i32 5>
+ store <2 x i32> %v0, ptr %out0
+ store <2 x i32> %v1, ptr %out1
+ store <2 x i32> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v4i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI13_0
+; CHECK-LE-IADISABLED-NEXT: ldr d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI13_2
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI13_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI13_1
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI13_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x9, :lo12:.LCPI13_2]
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ldr d0, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI13_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_0
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI13_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI13_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <12 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %v1 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %v2 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ store <4 x i16> %v0, ptr %out0
+ store <4 x i16> %v1, ptr %out1
+ store <4 x i16> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v8i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.8b, v1.8b, v2.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI14_0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI14_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI14_1
+; CHECK-LE-IADISABLED-NEXT: ldr d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI14_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI14_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI14_2]
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.8b, v1.8b, v2.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8b }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ldr d0, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI14_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI14_0
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI14_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI14_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI14_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI14_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <24 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %v0 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+ %v1 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+ %v2 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ store <8 x i8> %v0, ptr %out0
+ store <8 x i8> %v1, ptr %out1
+ store <8 x i8> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v2f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v3.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-LE-IADISABLED-NEXT: ext v1.8b, v0.8b, v2.8b, #4
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d2, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ldr d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov d2, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v4.s[1], v2.s[1]
+; CHECK-BE-IADISABLED-NEXT: ext v2.8b, v0.8b, v3.8b, #4
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <6 x float>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 0, i32 3>
+ %v1 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 1, i32 4>
+ %v2 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 2, i32 5>
+ store <2 x float> %v0, ptr %out0
+ store <2 x float> %v1, ptr %out1
+ store <2 x float> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v4f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI16_0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI16_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI16_1
+; CHECK-LE-IADISABLED-NEXT: ldr d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI16_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI16_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI16_2]
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ldr d0, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI16_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI16_0
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI16_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI16_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI16_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI16_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <12 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %v1 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %v2 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ store <4 x half> %v0, ptr %out0
+ store <4 x half> %v1, ptr %out1
+ store <4 x half> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v2i64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2i64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2i64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q1, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ext v3.16b, v1.16b, v0.16b, #8
+; CHECK-LE-IADISABLED-NEXT: mov v1.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v0.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <6 x i64>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 0, i32 3>
+ %v1 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 1, i32 4>
+ %v2 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 2, i32 5>
+ store <2 x i64> %v0, ptr %out0
+ store <2 x i64> %v1, ptr %out1
+ store <2 x i64> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v4i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: mov v1.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[1], v0.s[3]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[2], v2.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[2], v4.s[0]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[3], v4.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <12 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %v1 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %v2 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ store <4 x i32> %v0, ptr %out0
+ store <4 x i32> %v1, ptr %out1
+ store <4 x i32> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v8i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI19_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI19_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI19_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI19_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_3
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI19_4
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI19_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI19_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v2.16b, v3.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI19_5]
+; CHECK-LE-IADISABLED-NEXT: str q4, [x1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: str q5, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_0
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI19_2
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI19_2
+; CHECK-BE-IADISABLED-NEXT: add x10, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_3
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x10]
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_5
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v2.16b, v3.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.16b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.16b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <24 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+ %v1 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+ %v2 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ store <8 x i16> %v0, ptr %out0
+ store <8 x i16> %v1, ptr %out1
+ store <8 x i16> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v16i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v16i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.16b, v1.16b, v2.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v16i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI20_0]
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI20_2
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI20_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI20_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_3
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI20_4
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI20_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI20_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI20_5]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: str q5, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.16b, v1.16b, v2.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.16b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.16b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.16b }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI20_0
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI20_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI20_1
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI20_1
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_3
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_5
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.16b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <48 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %v0 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
+ <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21,
+ i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+ %v1 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
+ <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22,
+ i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+ %v2 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
+ <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23,
+ i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+ store <16 x i8> %v0, ptr %out0
+ store <16 x i8> %v1, ptr %out1
+ store <16 x i8> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v2f64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2f64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2f64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q1, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ext v3.16b, v1.16b, v0.16b, #8
+; CHECK-LE-IADISABLED-NEXT: mov v1.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v0.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <6 x double>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 0, i32 3>
+ %v1 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 1, i32 4>
+ %v2 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 2, i32 5>
+ store <2 x double> %v0, ptr %out0
+ store <2 x double> %v1, ptr %out1
+ store <2 x double> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v4f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: mov v1.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[1], v0.s[3]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[2], v2.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[2], v4.s[0]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[3], v4.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <12 x float>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %v1 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %v2 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ store <4 x float> %v0, ptr %out0
+ store <4 x float> %v1, ptr %out1
+ store <4 x float> %v2, ptr %out2
+ ret void
+}
+
+define void @test_ldnp_interleaved_load3_v8f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI23_0]
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI23_2
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI23_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI23_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_3
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI23_4
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI23_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI23_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI23_5]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: str q5, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x3]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load3_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load3_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI23_0
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI23_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI23_1
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI23_1
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_3
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_5
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.16b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <24 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+ %v1 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+ %v2 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ store <8 x half> %v0, ptr %out0
+ store <8 x half> %v1, ptr %out1
+ store <8 x half> %v2, ptr %out2
+ ret void
+}
+
+; Test conservative lowering of a ld4 matching patterns
+
+define void @test_ldnp_interleaved_load4_v2i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str d3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v3.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v2.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: str d3, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d4, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x3]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.2s }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v2.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v3.2s, v1.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v4.2s, v1.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2s }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 0, i32 4>
+ %v1 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 1, i32 5>
+ %v2 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 2, i32 6>
+ %v3 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 3, i32 7>
+ store <2 x i32> %v0, ptr %out0
+ store <2 x i32> %v1, ptr %out1
+ store <2 x i32> %v2, ptr %out2
+ store <2 x i32> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v4i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str d3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI25_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI25_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI25_1
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI25_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI25_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI25_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI25_3
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI25_3]
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d4, [x3]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.4h }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %v1 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %v2 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %v3 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ store <4 x i16> %v0, ptr %out0
+ store <4 x i16> %v1, ptr %out1
+ store <4 x i16> %v2, ptr %out2
+ store <4 x i16> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v8i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str d3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI26_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI26_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI26_1
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI26_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI26_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI26_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI26_3
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI26_3]
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d4, [x3]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8b }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.8b }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <32 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %v0 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+ %v1 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+ %v2 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+ %v3 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ store <8 x i8> %v0, ptr %out0
+ store <8 x i8> %v1, ptr %out1
+ store <8 x i8> %v2, ptr %out2
+ store <8 x i8> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v2f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str d3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v3.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v2.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: str d3, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d4, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x3]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.2s }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v2.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v3.2s, v1.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v4.2s, v1.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2s }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x float>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 0, i32 4>
+ %v1 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 1, i32 5>
+ %v2 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 2, i32 6>
+ %v3 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 3, i32 7>
+ store <2 x float> %v0, ptr %out0
+ store <2 x float> %v1, ptr %out1
+ store <2 x float> %v2, ptr %out2
+ store <2 x float> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v4f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str d2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str d3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI28_0
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q2, [x8, :lo12:.LCPI28_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI28_1
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI28_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI28_2
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI28_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI28_3
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI28_3]
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d3, [x2]
+; CHECK-LE-IADISABLED-NEXT: str d4, [x3]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.4h }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.8b }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %v1 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %v2 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %v3 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ store <4 x half> %v0, ptr %out0
+ store <4 x half> %v1, ptr %out1
+ store <4 x half> %v2, ptr %out2
+ store <4 x half> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v2i64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2i64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2i64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q2, q3, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v4.2d, v2.2d, v0.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v2.2d, v0.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v3.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v1.2d, v3.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q4, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.2d }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v4.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v3.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x i64>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 0, i32 4>
+ %v1 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 1, i32 5>
+ %v2 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 2, i32 6>
+ %v3 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 3, i32 7>
+ store <2 x i64> %v0, ptr %out0
+ store <2 x i64> %v1, ptr %out1
+ store <2 x i64> %v2, ptr %out2
+ store <2 x i64> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v4i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q3, q4, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v17.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v5.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v6.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v16.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: ext v7.16b, v0.16b, v2.16b, #8
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v5.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: mov v16.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v17.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v6.d[1], v7.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q6, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q16, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q4, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.4s }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v4.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: trn1 v17.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v5.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v6.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v16.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: ext v7.16b, v1.16b, v4.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v5.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.4s, v16.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.4s, v17.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov v6.d[1], v4.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v5.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v7.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v6.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2d }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %v1 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %v2 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %v3 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ store <4 x i32> %v0, ptr %out0
+ store <4 x i32> %v1, ptr %out1
+ store <4 x i32> %v2, ptr %out2
+ store <4 x i32> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v8i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI31_1
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI31_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_2
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI31_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x8, :lo12:.LCPI31_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_3
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI31_4
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x8, :lo12:.LCPI31_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_5
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x9, :lo12:.LCPI31_4]
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x8, :lo12:.LCPI31_5]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_6
+; CHECK-LE-IADISABLED-NEXT: ldr q18, [x8, :lo12:.LCPI31_6]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_7
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q19, [x8, :lo12:.LCPI31_7]
+; CHECK-LE-IADISABLED-NEXT: ldnp q2, q3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v0.16b, v1.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v17.16b, { v0.16b, v1.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v6.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v18.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v19.16b
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v17.d[1], v16.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q5, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q7, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q17, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.8h }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v16.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v17.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_6
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_6
+; CHECK-BE-IADISABLED-NEXT: adrp x10, .LCPI31_0
+; CHECK-BE-IADISABLED-NEXT: add x10, x10, :lo12:.LCPI31_0
+; CHECK-BE-IADISABLED-NEXT: add x11, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_1
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v18.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_7
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_7
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x10]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x11]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v19.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v3.16b, v4.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v17.16b, { v3.16b, v4.16b }, v17.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v1.16b, v2.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v16.16b, { v1.16b, v2.16b }, v16.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v18.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v19.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.16b, v16.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.16b, v17.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v7.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <32 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+ %v1 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+ %v2 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+ %v3 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ store <8 x i16> %v0, ptr %out0
+ store <8 x i16> %v1, ptr %out1
+ store <8 x i16> %v2, ptr %out2
+ store <8 x i16> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v16i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v16i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v16i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI32_1
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI32_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_2
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI32_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x8, :lo12:.LCPI32_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_3
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI32_4
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x8, :lo12:.LCPI32_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_5
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x9, :lo12:.LCPI32_4]
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x8, :lo12:.LCPI32_5]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_6
+; CHECK-LE-IADISABLED-NEXT: ldr q18, [x8, :lo12:.LCPI32_6]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_7
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q19, [x8, :lo12:.LCPI32_7]
+; CHECK-LE-IADISABLED-NEXT: ldnp q2, q3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v0.16b, v1.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v17.16b, { v0.16b, v1.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v6.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v18.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v19.16b
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v17.d[1], v16.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q5, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q7, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q17, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.16b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.16b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.16b }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.16b }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v16.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v17.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_6
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_6
+; CHECK-BE-IADISABLED-NEXT: ld1 { v18.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_7
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_7
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v19.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v0.16b, v1.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v17.16b, { v0.16b, v1.16b }, v17.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v18.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v19.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.16b, v17.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.16b, v16.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v7.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <64 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %v0 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
+ <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28,
+ i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+ %v1 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
+ <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29,
+ i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+ %v2 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
+ <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30,
+ i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+ %v3 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
+ <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31,
+ i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+ store <16 x i8> %v0, ptr %out0
+ store <16 x i8> %v1, ptr %out1
+ store <16 x i8> %v2, ptr %out2
+ store <16 x i8> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v2f64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2f64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2f64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q2, q3, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v4.2d, v2.2d, v0.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v2.2d, v0.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v3.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v1.2d, v3.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q4, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q2, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.2d }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.2d }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v4.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v3.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2d }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x double>, ptr %ptr, align 8, !nontemporal !0
+ %v0 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 0, i32 4>
+ %v1 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 1, i32 5>
+ %v2 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 2, i32 6>
+ %v3 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 3, i32 7>
+ store <2 x double> %v0, ptr %out0
+ store <2 x double> %v1, ptr %out1
+ store <2 x double> %v2, ptr %out2
+ store <2 x double> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v4f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ldnp q3, q4, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v17.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v5.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v6.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v16.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.4s, v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: ext v7.16b, v0.16b, v2.16b, #8
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v5.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: mov v16.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v17.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v6.d[1], v7.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q6, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q16, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q4, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q1, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.4s }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.4s }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v4.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: trn1 v17.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v5.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v6.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v16.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: ext v7.16b, v1.16b, v4.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v5.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.4s, v16.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.4s, v17.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov v6.d[1], v4.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v5.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v7.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x float>, ptr %ptr, align 4, !nontemporal !0
+ %v0 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %v1 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %v2 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %v3 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ store <4 x float> %v0, ptr %out0
+ store <4 x float> %v1, ptr %out1
+ store <4 x float> %v2, ptr %out2
+ store <4 x float> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load4_v8f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: str q2, [x3]
+; CHECK-LE-IAENABLED-NEXT: str q3, [x4]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI35_1
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI35_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_2
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI35_1]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x8, :lo12:.LCPI35_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_3
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI35_4
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x8, :lo12:.LCPI35_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_5
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x9, :lo12:.LCPI35_4]
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x8, :lo12:.LCPI35_5]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_6
+; CHECK-LE-IADISABLED-NEXT: ldr q18, [x8, :lo12:.LCPI35_6]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_7
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ldr q19, [x8, :lo12:.LCPI35_7]
+; CHECK-LE-IADISABLED-NEXT: ldnp q2, q3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v0.16b, v1.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v17.16b, { v0.16b, v1.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v6.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v18.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v19.16b
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v17.d[1], v16.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-LE-IADISABLED-NEXT: str q5, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q7, [x2]
+; CHECK-LE-IADISABLED-NEXT: str q17, [x3]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x4]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load4_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: st1 { v2.8h }, [x3]
+; CHECK-BE-IAENABLED-NEXT: st1 { v3.8h }, [x4]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load4_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_2
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_4
+; CHECK-BE-IADISABLED-NEXT: ld1 { v16.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v17.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_6
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_6
+; CHECK-BE-IADISABLED-NEXT: ld1 { v18.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_7
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_7
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v19.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v0.16b, v1.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v17.16b, { v0.16b, v1.16b }, v17.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v18.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v19.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.16b, v17.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.16b, v16.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v5.d[1], v4.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v3.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.8h, v5.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.8h, v7.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8h }, [x3]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x4]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <32 x half>, ptr %ptr, align 2, !nontemporal !0
+ %v0 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+ %v1 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+ %v2 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+ %v3 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ store <8 x half> %v0, ptr %out0
+ store <8 x half> %v1, ptr %out1
+ store <8 x half> %v2, ptr %out2
+ store <8 x half> %v3, ptr %out3
+ ret void
+}
+
+define void @test_ldnp_interleaved_load2_v2i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <4 x i32>, ptr %ptr, align 4, !nontemporal !0
+ %deinterleaved = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %loaded)
+ %v0 = extractvalue { <2 x i32>, <2 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i32>, <2 x i32> } %deinterleaved, 1
+ store <2 x i32> %v0, ptr %out0
+ store <2 x i32> %v1, ptr %out1
+ ret void
+}
+
+
+define void @test_ldnp_interleaved_load2_v4i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: xtn v1.4h, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <8 x i16>, ptr %ptr, align 2, !nontemporal !0
+ %deinterleaved = call { <4 x i16>, <4 x i16> } @llvm.vector.deinterleave2.v8i16(<8 x i16> %loaded)
+ %v0 = extractvalue { <4 x i16>, <4 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i16>, <4 x i16> } %deinterleaved, 1
+ store <4 x i16> %v0, ptr %out0
+ store <4 x i16> %v1, ptr %out1
+ ret void
+}
+
+
+define void @test_ldnp_interleaved_load2_v8i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: xtn v1.8b, v0.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i8_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.16b, v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <16 x i8>, ptr %ptr, align 1, !nontemporal !0
+ %deinterleaved = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> %loaded)
+ %v0 = extractvalue { <8 x i8>, <8 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i8>, <8 x i8> } %deinterleaved, 1
+ store <8 x i8> %v0, ptr %out0
+ store <8 x i8> %v1, ptr %out1
+ ret void
+}
+
+
+define void @test_ldnp_interleaved_load2_v2f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-LE-IADISABLED-NEXT: str d2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %loaded = load <4 x float>, ptr %ptr, align 4, !nontemporal !0
+ %deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %loaded)
+ %v0 = extractvalue { <2 x float>, <2 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x float>, <2 x float> } %deinterleaved, 1
+ store <2 x float> %v0, ptr %out0
+ store <2 x float> %v1, ptr %out1
+ ret void
+}
+
+
+define void @test_ldnp_interleaved_load2_v4f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str d0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str d1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4f16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.4h, v1.4h }, [x0]
-; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
-; CHECK-BE-NEXT: st1 { v1.4h }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-LE-IADISABLED-NEXT: str d1, [x1]
+; CHECK-LE-IADISABLED-NEXT: str d0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4h, v1.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v1.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <8 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <8 x half> %loaded, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %v1 = shufflevector <8 x half> %loaded, <8 x half> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %deinterleaved = call { <4 x half>, <4 x half> } @llvm.vector.deinterleave2.v8f16(<8 x half> %loaded)
+ %v0 = extractvalue { <4 x half>, <4 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x half>, <4 x half> } %deinterleaved, 1
store <4 x half> %v0, ptr %out0
store <4 x half> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v2i64(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2i64:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v2i64_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2i64:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
-; CHECK-BE-NEXT: st1 { v1.2d }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2i64_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <4 x i64>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <4 x i64> %loaded, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
- %v1 = shufflevector <4 x i64> %loaded, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+ %deinterleaved = call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %loaded)
+ %v0 = extractvalue { <2 x i64>, <2 x i64> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i64>, <2 x i64> } %deinterleaved, 1
store <2 x i64> %v0, ptr %out0
store <2 x i64> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v4i32(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4i32:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v4i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4i32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
-; CHECK-BE-NEXT: st1 { v1.4s }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4i32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <8 x i32>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %v1 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %deinterleaved = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %loaded)
+ %v0 = extractvalue { <4 x i32>, <4 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32> } %deinterleaved, 1
store <4 x i32> %v0, ptr %out0
store <4 x i32> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v8i16(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8i16:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v8i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8i16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
-; CHECK-BE-NEXT: st1 { v1.8h }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8i16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <16 x i16>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
- %v1 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %deinterleaved = call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %loaded)
+ %v0 = extractvalue { <8 x i16>, <8 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i16>, <8 x i16> } %deinterleaved, 1
store <8 x i16> %v0, ptr %out0
store <8 x i16> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v16i8(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v16i8:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.16b, v1.16b }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v16i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v16i8:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.16b, v1.16b }, [x0]
-; CHECK-BE-NEXT: st1 { v0.16b }, [x1]
-; CHECK-BE-NEXT: st1 { v1.16b }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.16b }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.16b }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v16i8_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <32 x i8>, ptr %ptr, align 1, !nontemporal !0
- %v0 = shufflevector <32 x i8> %loaded, <32 x i8> poison,
- <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14,
- i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
- %v1 = shufflevector <32 x i8> %loaded, <32 x i8> poison,
- <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15,
- i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+ %deinterleaved = call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %loaded)
+ %v0 = extractvalue { <16 x i8>, <16 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <16 x i8>, <16 x i8> } %deinterleaved, 1
store <16 x i8> %v0, ptr %out0
store <16 x i8> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v2f64(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v2f64:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v2f64_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v2f64:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.2d, v1.2d }, [x0]
-; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
-; CHECK-BE-NEXT: st1 { v1.2d }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.2d }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.2d }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v2f64_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <4 x double>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <4 x double> %loaded, <4 x double> poison, <2 x i32> <i32 0, i32 2>
- %v1 = shufflevector <4 x double> %loaded, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+ %deinterleaved = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %loaded)
+ %v0 = extractvalue { <2 x double>, <2 x double> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x double>, <2 x double> } %deinterleaved, 1
store <2 x double> %v0, ptr %out0
store <2 x double> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v4f32(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v4f32:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v4f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v4f32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.4s, v1.4s }, [x0]
-; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
-; CHECK-BE-NEXT: st1 { v1.4s }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.4s }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.4s }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v4f32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <8 x float>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <8 x float> %loaded, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %v1 = shufflevector <8 x float> %loaded, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %deinterleaved = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %loaded)
+ %v0 = extractvalue { <4 x float>, <4 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x float>, <4 x float> } %deinterleaved, 1
store <4 x float> %v0, ptr %out0
store <4 x float> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load2_v8f16(ptr %ptr, ptr %out0, ptr %out1) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load2_v8f16:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-LE-NEXT: str q0, [x1]
-; CHECK-LE-NEXT: str q1, [x2]
-; CHECK-LE-NEXT: ret
+
+define void @test_ldnp_interleaved_load2_v8f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1) {
+; CHECK-LE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: str q0, [x1]
+; CHECK-LE-IAENABLED-NEXT: str q1, [x2]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load2_v8f16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ld2 { v0.8h, v1.8h }, [x0]
-; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
-; CHECK-BE-NEXT: st1 { v1.8h }, [x2]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: ldnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: str q2, [x1]
+; CHECK-LE-IADISABLED-NEXT: str q0, [x2]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: st1 { v0.8h }, [x1]
+; CHECK-BE-IAENABLED-NEXT: st1 { v1.8h }, [x2]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_ldnp_interleaved_load2_v8f16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x2]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%loaded = load <16 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <16 x half> %loaded, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
- %v1 = shufflevector <16 x half> %loaded, <16 x half> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %deinterleaved = call { <8 x half>, <8 x half> } @llvm.vector.deinterleave2.v16f16(<16 x half> %loaded)
+ %v0 = extractvalue { <8 x half>, <8 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x half>, <8 x half> } %deinterleaved, 1
store <8 x half> %v0, ptr %out0
store <8 x half> %v1, ptr %out1
ret void
}
-define void @test_ldnp_interleaved_load3_v2i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2i32:
+
+define void @test_ldnp_interleaved_load3_v2i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -293,7 +2967,7 @@ define void @test_ldnp_interleaved_load3_v2i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2i32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
@@ -302,17 +2976,19 @@ define void @test_ldnp_interleaved_load3_v2i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <6 x i32>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 0, i32 3>
- %v1 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 1, i32 4>
- %v2 = shufflevector <6 x i32> %loaded, <6 x i32> poison, <2 x i32> <i32 2, i32 5>
+ %deinterleaved = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.vector.deinterleave3.v6i32(<6 x i32> %loaded)
+ %v0 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 2
store <2 x i32> %v0, ptr %out0
store <2 x i32> %v1, ptr %out1
store <2 x i32> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v4i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4i16:
+
+define void @test_ldnp_interleaved_load3_v4i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -320,7 +2996,7 @@ define void @test_ldnp_interleaved_load3_v4i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4i16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
@@ -329,17 +3005,19 @@ define void @test_ldnp_interleaved_load3_v4i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <12 x i16>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %v1 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %v2 = shufflevector <12 x i16> %loaded, <12 x i16> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %deinterleaved = call { <4 x i16>, <4 x i16>, <4 x i16> } @llvm.vector.deinterleave3.v12i16(<12 x i16> %loaded)
+ %v0 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 2
store <4 x i16> %v0, ptr %out0
store <4 x i16> %v1, ptr %out1
store <4 x i16> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v8i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8i8:
+
+define void @test_ldnp_interleaved_load3_v8i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.8b, v1.8b, v2.8b }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -347,7 +3025,7 @@ define void @test_ldnp_interleaved_load3_v8i8(ptr %ptr, ptr %out0, ptr %out1, pt
; CHECK-LE-NEXT: str d2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8i8:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.8b, v1.8b, v2.8b }, [x0]
; CHECK-BE-NEXT: st1 { v0.8b }, [x1]
@@ -356,17 +3034,19 @@ define void @test_ldnp_interleaved_load3_v8i8(ptr %ptr, ptr %out0, ptr %out1, pt
; CHECK-BE-NEXT: ret
entry:
%loaded = load <24 x i8>, ptr %ptr, align 1, !nontemporal !0
- %v0 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
- %v1 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
- %v2 = shufflevector <24 x i8> %loaded, <24 x i8> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ %deinterleaved = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.vector.deinterleave3.v24i8(<24 x i8> %loaded)
+ %v0 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 2
store <8 x i8> %v0, ptr %out0
store <8 x i8> %v1, ptr %out1
store <8 x i8> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v2f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2f32:
+
+define void @test_ldnp_interleaved_load3_v2f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -374,7 +3054,7 @@ define void @test_ldnp_interleaved_load3_v2f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2f32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
@@ -383,17 +3063,19 @@ define void @test_ldnp_interleaved_load3_v2f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <6 x float>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 0, i32 3>
- %v1 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 1, i32 4>
- %v2 = shufflevector <6 x float> %loaded, <6 x float> poison, <2 x i32> <i32 2, i32 5>
+ %deinterleaved = call { <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave3.v6f32(<6 x float> %loaded)
+ %v0 = extractvalue { <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 2
store <2 x float> %v0, ptr %out0
store <2 x float> %v1, ptr %out1
store <2 x float> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v4f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4f16:
+
+define void @test_ldnp_interleaved_load3_v4f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -401,7 +3083,7 @@ define void @test_ldnp_interleaved_load3_v4f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4f16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
@@ -410,17 +3092,19 @@ define void @test_ldnp_interleaved_load3_v4f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <12 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %v1 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %v2 = shufflevector <12 x half> %loaded, <12 x half> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %deinterleaved = call { <4 x half>, <4 x half>, <4 x half> } @llvm.vector.deinterleave3.v12f16(<12 x half> %loaded)
+ %v0 = extractvalue { <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 2
store <4 x half> %v0, ptr %out0
store <4 x half> %v1, ptr %out1
store <4 x half> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v2i64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2i64:
+
+define void @test_ldnp_interleaved_load3_v2i64_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2i64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -428,7 +3112,7 @@ define void @test_ldnp_interleaved_load3_v2i64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2i64:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2i64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
@@ -437,17 +3121,19 @@ define void @test_ldnp_interleaved_load3_v2i64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <6 x i64>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 0, i32 3>
- %v1 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 1, i32 4>
- %v2 = shufflevector <6 x i64> %loaded, <6 x i64> poison, <2 x i32> <i32 2, i32 5>
+ %deinterleaved = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64> %loaded)
+ %v0 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 2
store <2 x i64> %v0, ptr %out0
store <2 x i64> %v1, ptr %out1
store <2 x i64> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v4i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4i32:
+
+define void @test_ldnp_interleaved_load3_v4i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -455,7 +3141,7 @@ define void @test_ldnp_interleaved_load3_v4i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4i32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
@@ -464,17 +3150,19 @@ define void @test_ldnp_interleaved_load3_v4i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <12 x i32>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %v1 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %v2 = shufflevector <12 x i32> %loaded, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %deinterleaved = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave3.v12i32(<12 x i32> %loaded)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 2
store <4 x i32> %v0, ptr %out0
store <4 x i32> %v1, ptr %out1
store <4 x i32> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v8i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8i16:
+
+define void @test_ldnp_interleaved_load3_v8i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -482,7 +3170,7 @@ define void @test_ldnp_interleaved_load3_v8i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8i16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
@@ -491,17 +3179,19 @@ define void @test_ldnp_interleaved_load3_v8i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <24 x i16>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
- %v1 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
- %v2 = shufflevector <24 x i16> %loaded, <24 x i16> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ %deinterleaved = call { <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave3.v24i16(<24 x i16> %loaded)
+ %v0 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 2
store <8 x i16> %v0, ptr %out0
store <8 x i16> %v1, ptr %out1
store <8 x i16> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v16i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v16i8:
+
+define void @test_ldnp_interleaved_load3_v16i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v16i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.16b, v1.16b, v2.16b }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -509,7 +3199,7 @@ define void @test_ldnp_interleaved_load3_v16i8(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v16i8:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v16i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.16b, v1.16b, v2.16b }, [x0]
; CHECK-BE-NEXT: st1 { v0.16b }, [x1]
@@ -518,23 +3208,19 @@ define void @test_ldnp_interleaved_load3_v16i8(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <48 x i8>, ptr %ptr, align 1, !nontemporal !0
- %v0 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
- <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21,
- i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
- %v1 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
- <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22,
- i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
- %v2 = shufflevector <48 x i8> %loaded, <48 x i8> poison,
- <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23,
- i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+ %deinterleaved = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave3.v48i8(<48 x i8> %loaded)
+ %v0 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 1
+ %v2 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 2
store <16 x i8> %v0, ptr %out0
store <16 x i8> %v1, ptr %out1
store <16 x i8> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v2f64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2f64:
+
+define void @test_ldnp_interleaved_load3_v2f64_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v2f64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -542,7 +3228,7 @@ define void @test_ldnp_interleaved_load3_v2f64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2f64:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v2f64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
@@ -551,17 +3237,19 @@ define void @test_ldnp_interleaved_load3_v2f64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <6 x double>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 0, i32 3>
- %v1 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 1, i32 4>
- %v2 = shufflevector <6 x double> %loaded, <6 x double> poison, <2 x i32> <i32 2, i32 5>
+ %deinterleaved = call { <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave3.v6f64(<6 x double> %loaded)
+ %v0 = extractvalue { <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 2
store <2 x double> %v0, ptr %out0
store <2 x double> %v1, ptr %out1
store <2 x double> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v4f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4f32:
+
+define void @test_ldnp_interleaved_load3_v4f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v4f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -569,7 +3257,7 @@ define void @test_ldnp_interleaved_load3_v4f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4f32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v4f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
@@ -578,17 +3266,19 @@ define void @test_ldnp_interleaved_load3_v4f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <12 x float>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %v1 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %v2 = shufflevector <12 x float> %loaded, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %deinterleaved = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %loaded)
+ %v0 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 2
store <4 x float> %v0, ptr %out0
store <4 x float> %v1, ptr %out1
store <4 x float> %v2, ptr %out2
ret void
}
-define void @test_ldnp_interleaved_load3_v8f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8f16:
+
+define void @test_ldnp_interleaved_load3_v8f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load3_v8f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -596,7 +3286,7 @@ define void @test_ldnp_interleaved_load3_v8f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q2, [x3]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8f16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load3_v8f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
@@ -605,19 +3295,19 @@ define void @test_ldnp_interleaved_load3_v8f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <24 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
- %v1 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
- %v2 = shufflevector <24 x half> %loaded, <24 x half> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ %deinterleaved = call { <8 x half>, <8 x half>, <8 x half> } @llvm.vector.deinterleave3.v24f16(<24 x half> %loaded)
+ %v0 = extractvalue { <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 2
store <8 x half> %v0, ptr %out0
store <8 x half> %v1, ptr %out1
store <8 x half> %v2, ptr %out2
ret void
}
-; Test conservative lowering of a ld4 matching patterns
-define void @test_ldnp_interleaved_load4_v2i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2i32:
+define void @test_ldnp_interleaved_load4_v2i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -626,7 +3316,7 @@ define void @test_ldnp_interleaved_load4_v2i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2i32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
@@ -636,10 +3326,11 @@ define void @test_ldnp_interleaved_load4_v2i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x i32>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 0, i32 4>
- %v1 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 1, i32 5>
- %v2 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 2, i32 6>
- %v3 = shufflevector <8 x i32> %loaded, <8 x i32> poison, <2 x i32> <i32 3, i32 7>
+ %deinterleaved = call { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.vector.deinterleave4.v8i32(<8 x i32> %loaded)
+ %v0 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 2
+ %v3 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %deinterleaved, 3
store <2 x i32> %v0, ptr %out0
store <2 x i32> %v1, ptr %out1
store <2 x i32> %v2, ptr %out2
@@ -647,8 +3338,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v4i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4i16:
+
+define void @test_ldnp_interleaved_load4_v4i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -657,7 +3349,7 @@ define void @test_ldnp_interleaved_load4_v4i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4i16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
@@ -667,10 +3359,11 @@ define void @test_ldnp_interleaved_load4_v4i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x i16>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %v1 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %v2 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %v3 = shufflevector <16 x i16> %loaded, <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %deinterleaved = call { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.vector.deinterleave4.v16i16(<16 x i16> %loaded)
+ %v0 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 2
+ %v3 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %deinterleaved, 3
store <4 x i16> %v0, ptr %out0
store <4 x i16> %v1, ptr %out1
store <4 x i16> %v2, ptr %out2
@@ -678,8 +3371,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v8i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8i8:
+
+define void @test_ldnp_interleaved_load4_v8i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -688,7 +3382,7 @@ define void @test_ldnp_interleaved_load4_v8i8(ptr %ptr, ptr %out0, ptr %out1, pt
; CHECK-LE-NEXT: str d3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8i8:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
; CHECK-BE-NEXT: st1 { v0.8b }, [x1]
@@ -698,10 +3392,11 @@ define void @test_ldnp_interleaved_load4_v8i8(ptr %ptr, ptr %out0, ptr %out1, pt
; CHECK-BE-NEXT: ret
entry:
%loaded = load <32 x i8>, ptr %ptr, align 1, !nontemporal !0
- %v0 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
- %v1 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
- %v2 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
- %v3 = shufflevector <32 x i8> %loaded, <32 x i8> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ %deinterleaved = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.vector.deinterleave4.v32i8(<32 x i8> %loaded)
+ %v0 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 2
+ %v3 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %deinterleaved, 3
store <8 x i8> %v0, ptr %out0
store <8 x i8> %v1, ptr %out1
store <8 x i8> %v2, ptr %out2
@@ -709,8 +3404,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v2f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2f32:
+
+define void @test_ldnp_interleaved_load4_v2f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -719,7 +3415,7 @@ define void @test_ldnp_interleaved_load4_v2f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2f32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-BE-NEXT: st1 { v0.2s }, [x1]
@@ -729,10 +3425,11 @@ define void @test_ldnp_interleaved_load4_v2f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x float>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 0, i32 4>
- %v1 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 1, i32 5>
- %v2 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 2, i32 6>
- %v3 = shufflevector <8 x float> %loaded, <8 x float> poison, <2 x i32> <i32 3, i32 7>
+ %deinterleaved = call { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.vector.deinterleave4.v8f32(<8 x float> %loaded)
+ %v0 = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 2
+ %v3 = extractvalue { <2 x float>, <2 x float>, <2 x float>, <2 x float> } %deinterleaved, 3
store <2 x float> %v0, ptr %out0
store <2 x float> %v1, ptr %out1
store <2 x float> %v2, ptr %out2
@@ -740,8 +3437,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v4f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4f16:
+
+define void @test_ldnp_interleaved_load4_v4f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-LE-NEXT: str d0, [x1]
@@ -750,7 +3448,7 @@ define void @test_ldnp_interleaved_load4_v4f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str d3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4f16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-BE-NEXT: st1 { v0.4h }, [x1]
@@ -760,10 +3458,11 @@ define void @test_ldnp_interleaved_load4_v4f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %v1 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %v2 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %v3 = shufflevector <16 x half> %loaded, <16 x half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %deinterleaved = call { <4 x half>, <4 x half>, <4 x half>, <4 x half> } @llvm.vector.deinterleave4.v16f16(<16 x half> %loaded)
+ %v0 = extractvalue { <4 x half>, <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x half>, <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x half>, <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 2
+ %v3 = extractvalue { <4 x half>, <4 x half>, <4 x half>, <4 x half> } %deinterleaved, 3
store <4 x half> %v0, ptr %out0
store <4 x half> %v1, ptr %out1
store <4 x half> %v2, ptr %out2
@@ -771,8 +3470,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v2i64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2i64:
+
+define void @test_ldnp_interleaved_load4_v2i64_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2i64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -781,7 +3481,7 @@ define void @test_ldnp_interleaved_load4_v2i64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2i64:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2i64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
@@ -791,10 +3491,11 @@ define void @test_ldnp_interleaved_load4_v2i64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x i64>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 0, i32 4>
- %v1 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 1, i32 5>
- %v2 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 2, i32 6>
- %v3 = shufflevector <8 x i64> %loaded, <8 x i64> poison, <2 x i32> <i32 3, i32 7>
+ %deinterleaved = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave4.v8i64(<8 x i64> %loaded)
+ %v0 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 2
+ %v3 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %deinterleaved, 3
store <2 x i64> %v0, ptr %out0
store <2 x i64> %v1, ptr %out1
store <2 x i64> %v2, ptr %out2
@@ -802,8 +3503,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v4i32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4i32:
+
+define void @test_ldnp_interleaved_load4_v4i32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -812,7 +3514,7 @@ define void @test_ldnp_interleaved_load4_v4i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4i32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
@@ -822,10 +3524,11 @@ define void @test_ldnp_interleaved_load4_v4i32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x i32>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %v1 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %v2 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %v3 = shufflevector <16 x i32> %loaded, <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %deinterleaved = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4.v16i32(<16 x i32> %loaded)
+ %v0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 2
+ %v3 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %deinterleaved, 3
store <4 x i32> %v0, ptr %out0
store <4 x i32> %v1, ptr %out1
store <4 x i32> %v2, ptr %out2
@@ -833,8 +3536,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v8i16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8i16:
+
+define void @test_ldnp_interleaved_load4_v8i16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -843,7 +3547,7 @@ define void @test_ldnp_interleaved_load4_v8i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8i16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
@@ -853,10 +3557,11 @@ define void @test_ldnp_interleaved_load4_v8i16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <32 x i16>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
- %v1 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
- %v2 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
- %v3 = shufflevector <32 x i16> %loaded, <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ %deinterleaved = call { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.vector.deinterleave4.v32i16(<32 x i16> %loaded)
+ %v0 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 2
+ %v3 = extractvalue { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } %deinterleaved, 3
store <8 x i16> %v0, ptr %out0
store <8 x i16> %v1, ptr %out1
store <8 x i16> %v2, ptr %out2
@@ -864,8 +3569,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v16i8(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v16i8:
+
+define void @test_ldnp_interleaved_load4_v16i8_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v16i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -874,7 +3580,7 @@ define void @test_ldnp_interleaved_load4_v16i8(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v16i8:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v16i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
; CHECK-BE-NEXT: st1 { v0.16b }, [x1]
@@ -884,18 +3590,11 @@ define void @test_ldnp_interleaved_load4_v16i8(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <64 x i8>, ptr %ptr, align 1, !nontemporal !0
- %v0 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
- <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28,
- i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
- %v1 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
- <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29,
- i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
- %v2 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
- <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30,
- i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
- %v3 = shufflevector <64 x i8> %loaded, <64 x i8> poison,
- <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31,
- i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+ %deinterleaved = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.vector.deinterleave4.v64i8(<64 x i8> %loaded)
+ %v0 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 0
+ %v1 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 1
+ %v2 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 2
+ %v3 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %deinterleaved, 3
store <16 x i8> %v0, ptr %out0
store <16 x i8> %v1, ptr %out1
store <16 x i8> %v2, ptr %out2
@@ -903,8 +3602,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v2f64(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2f64:
+
+define void @test_ldnp_interleaved_load4_v2f64_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v2f64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -913,7 +3613,7 @@ define void @test_ldnp_interleaved_load4_v2f64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2f64:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v2f64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-BE-NEXT: st1 { v0.2d }, [x1]
@@ -923,10 +3623,11 @@ define void @test_ldnp_interleaved_load4_v2f64(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <8 x double>, ptr %ptr, align 8, !nontemporal !0
- %v0 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 0, i32 4>
- %v1 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 1, i32 5>
- %v2 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 2, i32 6>
- %v3 = shufflevector <8 x double> %loaded, <8 x double> poison, <2 x i32> <i32 3, i32 7>
+ %deinterleaved = call { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.vector.deinterleave4.v8f64(<8 x double> %loaded)
+ %v0 = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 0
+ %v1 = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 1
+ %v2 = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 2
+ %v3 = extractvalue { <2 x double>, <2 x double>, <2 x double>, <2 x double> } %deinterleaved, 3
store <2 x double> %v0, ptr %out0
store <2 x double> %v1, ptr %out1
store <2 x double> %v2, ptr %out2
@@ -934,8 +3635,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v4f32(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4f32:
+
+define void @test_ldnp_interleaved_load4_v4f32_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v4f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -944,7 +3646,7 @@ define void @test_ldnp_interleaved_load4_v4f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4f32:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v4f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-BE-NEXT: st1 { v0.4s }, [x1]
@@ -954,10 +3656,11 @@ define void @test_ldnp_interleaved_load4_v4f32(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <16 x float>, ptr %ptr, align 4, !nontemporal !0
- %v0 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %v1 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %v2 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %v3 = shufflevector <16 x float> %loaded, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %deinterleaved = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %loaded)
+ %v0 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 0
+ %v1 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 1
+ %v2 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 2
+ %v3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved, 3
store <4 x float> %v0, ptr %out0
store <4 x float> %v1, ptr %out1
store <4 x float> %v2, ptr %out2
@@ -965,8 +3668,9 @@ entry:
ret void
}
-define void @test_ldnp_interleaved_load4_v8f16(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
-; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8f16:
+
+define void @test_ldnp_interleaved_load4_v8f16_intrinsic(ptr %ptr, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+; CHECK-LE-LABEL: test_ldnp_interleaved_load4_v8f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-LE-NEXT: str q0, [x1]
@@ -975,7 +3679,7 @@ define void @test_ldnp_interleaved_load4_v8f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-LE-NEXT: str q3, [x4]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8f16:
+; CHECK-BE-LABEL: test_ldnp_interleaved_load4_v8f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-BE-NEXT: st1 { v0.8h }, [x1]
@@ -985,10 +3689,11 @@ define void @test_ldnp_interleaved_load4_v8f16(ptr %ptr, ptr %out0, ptr %out1, p
; CHECK-BE-NEXT: ret
entry:
%loaded = load <32 x half>, ptr %ptr, align 2, !nontemporal !0
- %v0 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
- %v1 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
- %v2 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
- %v3 = shufflevector <32 x half> %loaded, <32 x half> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ %deinterleaved = call { <8 x half>, <8 x half>, <8 x half>, <8 x half> } @llvm.vector.deinterleave4.v32f16(<32 x half> %loaded)
+ %v0 = extractvalue { <8 x half>, <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 0
+ %v1 = extractvalue { <8 x half>, <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 1
+ %v2 = extractvalue { <8 x half>, <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 2
+ %v3 = extractvalue { <8 x half>, <8 x half>, <8 x half>, <8 x half> } %deinterleaved, 3
store <8 x half> %v0, ptr %out0
store <8 x half> %v1, ptr %out1
store <8 x half> %v2, ptr %out2
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
index 326cb4e1677b6..8793f55f72db3 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved-optsize.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefix CHECK-LE
-; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefix CHECK-BE
+; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IADISABLED
+; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IADISABLED
; Note: Currently, interleaved stores are only directly supported on AArch64 for 64 bit or 128 bit vector data.
@@ -8,19 +10,37 @@
; when optimizing for size.
define void @test_stnp_interleaved_store2_v2i32_Os(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #0 {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i32_Os:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32_Os:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x i32> %v0, <2 x i32> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -29,19 +49,34 @@ entry:
}
define void @test_stnp_interleaved_store2_v2i32_Oz(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #1 {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-LE-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-LE-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: b OUTLINED_FUNCTION_1
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x i32> %v0, <2 x i32> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -50,19 +85,37 @@ entry:
}
define void @test_stnp_interleaved_store2_v2i64_Os(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #0 {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i64_Os:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_Os:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x i64> %v0, <2 x i64> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -71,19 +124,20 @@ entry:
}
define void @test_stnp_interleaved_store2_v2i64_Oz(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #1 {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i64_Oz:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-LE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-LE-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; CHECK-LE-NEXT: ret
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: b OUTLINED_FUNCTION_0
;
; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_Oz:
; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-NEXT: b OUTLINED_FUNCTION_0
entry:
%shuffle = shufflevector <2 x i64> %v0, <2 x i64> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -91,7 +145,148 @@ entry:
ret void
}
+define void @test_stnp_interleaved_store2_v2i32_Os_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #0 {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Os_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
+ store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v2i32_Oz_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) #1 {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: str q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: b OUTLINED_FUNCTION_1
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_Oz_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
+ store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v2i64_Os_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #0 {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Os_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
+ store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v2i64_Oz_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) #1 {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: b OUTLINED_FUNCTION_0
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64_Oz_intrinsic:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: b OUTLINED_FUNCTION_0
+entry:
+ %interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
+ store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
!0 = !{ i32 1 }
attributes #0 = { optsize }
attributes #1 = { minsize optsize }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-LE: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
index 398ffc5d2c941..10dccca55585a 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-store-interleaved.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE
-; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE
+; RUN: llc < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64 | FileCheck %s --check-prefixes=CHECK-LE,CHECK-LE-IADISABLED
+; RUN: llc < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple aarch64_be | FileCheck %s --check-prefixes=CHECK-BE,CHECK-BE-IADISABLED
; Note: Currently, interleaved stores are only directly supported on AArch64 for 64 bit or 128 bit vector data.
; Thus, this test cannot cover lowering to `STNPSi`.
@@ -15,12 +17,22 @@ define void @test_stnp_interleaved_store2_v2i32(<2 x i32> %v0, <2 x i32> %v1, pt
; CHECK-LE-NEXT: stnp d0, d1, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x i32> %v0, <2 x i32> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -38,12 +50,22 @@ define void @test_stnp_interleaved_store2_v4i16(<4 x i16> %v0, <4 x i16> %v1, pt
; CHECK-LE-NEXT: stnp d0, d1, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4i16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.4h, v1.4h
-; CHECK-BE-NEXT: rev64 v1.4h, v0.4h
-; CHECK-BE-NEXT: st2 { v1.4h, v2.4h }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <4 x i16> %v0, <4 x i16> %v1,
<8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -61,12 +83,22 @@ define void @test_stnp_interleaved_store2_v8i8(<8 x i8> %v0, <8 x i8> %v1, ptr %
; CHECK-LE-NEXT: stnp d0, d1, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8i8:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.8b, v1.8b
-; CHECK-BE-NEXT: rev64 v1.8b, v0.8b
-; CHECK-BE-NEXT: st2 { v1.8b, v2.8b }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8b, v1.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8b, v0.8b
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8b, v2.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <8 x i8> %v0, <8 x i8> %v1,
<16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
@@ -85,12 +117,22 @@ define void @test_stnp_interleaved_store2_v2f32(<2 x float> %v0, <2 x float> %v1
; CHECK-LE-NEXT: stnp d0, d1, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2f32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.2s, v1.2s
-; CHECK-BE-NEXT: rev64 v1.2s, v0.2s
-; CHECK-BE-NEXT: st2 { v1.2s, v2.2s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x float> %v0, <2 x float> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -108,12 +150,22 @@ define void @test_stnp_interleaved_store2_v4f16(<4 x half> %v0, <4 x half> %v1,
; CHECK-LE-NEXT: stnp d0, d1, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4f16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v2.4h, v1.4h
-; CHECK-BE-NEXT: rev64 v1.4h, v0.4h
-; CHECK-BE-NEXT: st2 { v1.4h, v2.4h }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <4 x half> %v0, <4 x half> %v1,
<8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -129,12 +181,23 @@ define void @test_stnp_interleaved_store2_v2i64(<2 x i64> %v0, <2 x i64> %v1, pt
; CHECK-LE-NEXT: stnp q0, q2, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i64:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <2 x i64> %v0, <2 x i64> %v1,
<4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -150,14 +213,27 @@ define void @test_stnp_interleaved_store2_v4i32(<4 x i32> %v0, <4 x i32> %v1, pt
; CHECK-LE-NEXT: stnp q0, q2, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4i32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.4s, v2.4s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <4 x i32> %v0, <4 x i32> %v1,
<8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -173,14 +249,27 @@ define void @test_stnp_interleaved_store2_v8i16(<8 x i16> %v0, <8 x i16> %v1, pt
; CHECK-LE-NEXT: stnp q0, q2, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8i16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.8h, v2.8h }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
%shuffle = shufflevector <8 x i16> %v0, <8 x i16> %v1,
<16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
@@ -203,95 +292,2608 @@ define void @test_stnp_interleaved_store2_v16i8(<16 x i8> %v0, <16 x i8> %v1, pt
; CHECK-LE-NEXT: stnp q0, q2, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v16i8:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
-; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v3.16b, v2.16b, v1.16b, #8
-; CHECK-BE-NEXT: st2 { v2.16b, v3.16b }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v2.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v2.16b, v3.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI8_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI8_1
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI8_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI8_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %shuffle = shufflevector <16 x i8> %v0, <16 x i8> %v1,
+ <32 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
+ i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15,
+ i32 8, i32 16, i32 9, i32 17, i32 10, i32 18, i32 11, i32 19,
+ i32 12, i32 20, i32 13, i32 21, i32 14, i32 22, i32 15, i32 23>
+ store <32 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store2_v2f64(<2 x double> %v0, <2 x double> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2f64:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-LE-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %shuffle = shufflevector <2 x double> %v0, <2 x double> %v1,
+ <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ store <4 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store2_v4f32(<4 x float> %v0, <4 x float> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4f32:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %shuffle = shufflevector <4 x float> %v0, <4 x float> %v1,
+ <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+ store <8 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store2_v8f16(<8 x half> %v0, <8 x half> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v8f16:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %shuffle = shufflevector <8 x half> %v0, <8 x half> %v1,
+ <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
+ i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+ store <16 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: fmov d3, d0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[0], v1.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.s[3], v3.s[1]
+; CHECK-LE-IADISABLED-NEXT: str d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.2s, v2.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.2s, v3.2s, v4.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: fmov d3, d0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[0], v1.s[1]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v0.s[3], v3.s[1]
+; CHECK-BE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x i32> %v2, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %shuffle = shufflevector <4 x i32> %s0, <4 x i32> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+ store <6 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d3, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI13_1
+; CHECK-LE-IADISABLED-NEXT: fmov d2, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI13_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI13_0
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI13_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.4h, v2.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.4h, v3.4h, v4.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI13_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI13_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x i16> %v0, <4 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x i16> %v2, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <8 x i16> %s0, <8 x i16> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ store <12 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.8b, v1.8b, v2.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d3, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI14_1
+; CHECK-LE-IADISABLED-NEXT: fmov d2, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI14_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI14_0
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.8b, v2.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.8b, v1.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8b, v0.8b
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.8b, v3.8b, v4.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8b, v1.8b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.8b, v0.8b
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI14_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI14_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI14_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI14_1
+; CHECK-BE-IADISABLED-NEXT: mov v3.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: str d0, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x i8> %v2, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <16 x i8> %s0, <16 x i8> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+ store <24 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v2f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.2s, v1.2s, v2.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: fmov d3, d0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: mov v2.s[0], v1.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.s[3], v3.s[1]
+; CHECK-LE-IADISABLED-NEXT: str d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.2s, v2.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.2s, v3.2s, v4.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: fmov d3, d0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v2.s[0], v1.s[1]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: mov v0.s[3], v3.s[1]
+; CHECK-BE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x float> %v0, <2 x float> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x float> %v2, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %shuffle = shufflevector <4 x float> %s0, <4 x float> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+ store <6 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v4f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d3, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI16_1
+; CHECK-LE-IADISABLED-NEXT: fmov d2, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI16_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI16_0
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.4h, v2.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.4h, v3.4h, v4.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI16_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI16_0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI16_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI16_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: str d1, [x0, #16]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x half> %v2, <4 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <8 x half> %s0, <8 x half> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ store <12 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2i64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2i64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v3.2d, v1.2d, v2.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[0], v2.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov d2, v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d5, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d3, d2, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d5, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.2d, v3.2d, v4.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.2d, v1.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[0], v2.d[0]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x i64> %v0, <2 x i64> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x i64> %v2, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %shuffle = shufflevector <4 x i64> %s0, <4 x i64> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+ store <6 x i64> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: trn2 v3.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v5.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: ext v3.16b, v3.16b, v0.16b, #4
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v0.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[1], v2.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v5.s[2], v2.s[0]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; CHECK-LE-IADISABLED-NEXT: mov d0, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d0, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d5, d2, [x0]
+; CHECK-LE-IADISABLED-NEXT: stnp d3, d4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: trn1 v4.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v5.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v6.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v5.16b, v3.16b, #4
+; CHECK-BE-IADISABLED-NEXT: mov v4.d[1], v0.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: mov v0.s[1], v2.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[2], v2.s[0]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ store <12 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q1_q2 def $q1_q2
+; CHECK-LE-IADISABLED-NEXT: mov v4.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI19_2
+; CHECK-LE-IADISABLED-NEXT: mov v3.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI19_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_1
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI19_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI19_4
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI19_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI19_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_3
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI19_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI19_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v1.16b, v2.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI19_5]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d3, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d5, d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.8h, v3.8h, v4.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_0
+; CHECK-BE-IADISABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_1
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_2
+; CHECK-BE-IADISABLED-NEXT: ext v6.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_4
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI19_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI19_5
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <16 x i16> %s0, <16 x i16> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+ store <24 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v16i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v16i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v16i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q1_q2 def $q1_q2
+; CHECK-LE-IADISABLED-NEXT: mov v4.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI20_2
+; CHECK-LE-IADISABLED-NEXT: mov v3.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI20_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_1
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI20_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI20_4
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI20_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI20_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_3
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI20_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI20_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v1.16b, v2.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI20_5]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d3, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d5, d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.16b, v3.16b, v4.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_0
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_1
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_2
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_4
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v1.16b, v2.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI20_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI20_5
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v1.16b, v2.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <16 x i8> %v0, <16 x i8> %v1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+ %s1 = shufflevector <16 x i8> %v2, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <32 x i8> %s0, <32 x i8> %s1, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+ store <48 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v2f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2f64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2f64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v3.2d, v1.2d, v2.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[0], v2.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov d2, v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d5, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d3, d2, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d5, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.2d, v3.2d, v4.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x9, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.2d, v1.2d, v2.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[0], v2.d[0]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x9]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x double> %v2, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %shuffle = shufflevector <4 x double> %s0, <4 x double> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+ store <6 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v4f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: trn2 v3.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v5.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: ext v3.16b, v3.16b, v0.16b, #4
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v0.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; CHECK-LE-IADISABLED-NEXT: mov v1.s[1], v2.s[1]
+; CHECK-LE-IADISABLED-NEXT: mov v5.s[2], v2.s[0]
+; CHECK-LE-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; CHECK-LE-IADISABLED-NEXT: mov d0, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d0, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d5, d2, [x0]
+; CHECK-LE-IADISABLED-NEXT: stnp d3, d4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: trn1 v4.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v5.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v6.4s, v3.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v5.16b, v3.16b, #4
+; CHECK-BE-IADISABLED-NEXT: mov v4.d[1], v0.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: mov v0.s[1], v2.s[1]
+; CHECK-BE-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; CHECK-BE-IADISABLED-NEXT: mov v1.s[2], v2.s[0]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x float> %v2, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <8 x float> %s0, <8 x float> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ store <12 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store3_v8f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-LE-IAENABLED-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q1_q2 def $q1_q2
+; CHECK-LE-IADISABLED-NEXT: mov v4.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI23_2
+; CHECK-LE-IADISABLED-NEXT: mov v3.16b, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI23_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_1
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI23_2]
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI23_4
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI23_4]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI23_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_3
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI23_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI23_5
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v1.16b, v2.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q3, [x8, :lo12:.LCPI23_5]
+; CHECK-LE-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-LE-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d4, [x0]
+; CHECK-LE-IADISABLED-NEXT: mov d3, v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d5, d2, [x0, #16]
+; CHECK-LE-IADISABLED-NEXT: stnp d1, d3, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store3_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st3 { v2.8h, v3.8h, v4.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store3_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_0
+; CHECK-BE-IADISABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_1
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_2
+; CHECK-BE-IADISABLED-NEXT: ext v6.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_4
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_4
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI23_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI23_5
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x half> %v0, <8 x half> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x half> %v2, <8 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %shuffle = shufflevector <16 x half> %s0, <16 x half> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+ store <24 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+; Test conservative lowering of a st4 matching patterns
+
+define void @test_stnp_interleaved_store4_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v6.2s, v3.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v5.2s, v2.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.2s, v2.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.2s, v3.2s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x i32> %v2, <2 x i32> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %shuffle = shufflevector <4 x i32> %s0, <4 x i32> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+ store <8 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, <4 x i16> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d5, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI25_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI25_1
+; CHECK-LE-IADISABLED-NEXT: fmov d4, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI25_0]
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v3.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI25_1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v4.16b, v5.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: stnp q1, q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v6.4h, v3.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v5.4h, v2.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.4h, v4.4h, v5.4h, v6.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4h, v3.4h
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_1
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v1.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI25_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI25_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x i16> %v0, <4 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x i16> %v2, <4 x i16> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %shuffle = shufflevector <8 x i16> %s0, <8 x i16> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ store <16 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d5, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI26_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI26_1
+; CHECK-LE-IADISABLED-NEXT: fmov d4, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI26_0]
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v3.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI26_1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v4.16b, v5.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: stnp q1, q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v6.8b, v3.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v5.8b, v2.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.8b, v1.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.8b, v0.8b
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.8b, v4.8b, v5.8b, v6.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.8b, v0.8b
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.8b, v3.8b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_1
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.8b, v2.8b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8b, v1.8b
+; CHECK-BE-IADISABLED-NEXT: mov v5.d[1], v3.d[0]
+; CHECK-BE-IADISABLED-NEXT: mov v4.d[1], v0.d[0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI26_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v4.16b, v5.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x i8> %v2, <8 x i8> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %shuffle = shufflevector <16 x i8> %s0, <16 x i8> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+ store <32 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v2f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, <2 x float> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: uzp2 v1.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v6.2s, v3.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v5.2s, v2.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.2s, v2.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.2s, v3.2s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: uzp2 v1.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x float> %v0, <2 x float> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x float> %v2, <2 x float> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %shuffle = shufflevector <4 x float> %s0, <4 x float> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+ store <8 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v4f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, <4 x half> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: fmov d5, d2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI28_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI28_1
+; CHECK-LE-IADISABLED-NEXT: fmov d4, d0
+; CHECK-LE-IADISABLED-NEXT: ldr q0, [x8, :lo12:.LCPI28_0]
+; CHECK-LE-IADISABLED-NEXT: mov v5.d[1], v3.d[0]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x9, :lo12:.LCPI28_1]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v4.16b, v5.16b }, v0.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v4.16b, v5.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: stnp q1, q0, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v6.4h, v3.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v5.4h, v2.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v4.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.4h, v4.4h, v5.4h, v6.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4h, v2.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4h, v3.4h
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_1
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v1.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI28_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI28_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x half> %v2, <4 x half> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %shuffle = shufflevector <8 x half> %s0, <8 x half> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ store <16 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2i64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2i64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.2d, v2.2d, v3.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v5.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v3.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stnp q5, q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2i64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.2d, v4.2d, v5.2d, v6.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2i64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v4.2d, v2.2d, v3.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v3.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v5.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x i64> %v0, <2 x i64> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x i64> %v2, <2 x i64> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %shuffle = shufflevector <4 x i64> %s0, <4 x i64> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+ store <8 x i64> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4i32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4i32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v4.4s, v4.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; CHECK-LE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v16.d[1], v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp q7, q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q16, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4i32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.4s, v4.4s, v5.4s, v6.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4i32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v4.4s, v4.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.4s, v5.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.4s, v16.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov v6.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v7.d[1], v5.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: st1 { v6.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v7.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %shuffle = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ store <16 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8i16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8i16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI31_5
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q3 killed $q3 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI31_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_7
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI31_5]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x8, :lo12:.LCPI31_7]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI31_0
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x8, :lo12:.LCPI31_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_4
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v6.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x9, :lo12:.LCPI31_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI31_4]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_6
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x8, :lo12:.LCPI31_6]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI31_2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v2.16b, v3.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x8, :lo12:.LCPI31_2]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v5.4s, v5.4s
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v6.4s, v6.4s
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v1.4s, v5.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v4.4s, v6.4s, v16.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v3.4s, v3.4s, v7.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q4, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q3, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8i16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.8h, v3.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.8h, v4.8h, v5.8h, v6.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8i16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_3
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_3
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_2
+; CHECK-BE-IADISABLED-NEXT: ext v6.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_7
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_7
+; CHECK-BE-IADISABLED-NEXT: ext v5.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v17.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_5
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_6
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_6
+; CHECK-BE-IADISABLED-NEXT: ext v16.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI31_4
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI31_4
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v5.16b, v6.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI31_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI31_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v16.16b, v17.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: rev32 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v16.16b, v17.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v4.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v16.16b, v17.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v16.16b, v17.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v7.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v5.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v6.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v2.4s, v2.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v1.4s, v1.4s, v5.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v3.4s, v4.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x i16> %v2, <8 x i16> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %shuffle = shufflevector <16 x i16> %s0, <16 x i16> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+ store <32 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v16i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v16i8:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v16i8:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_0
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI32_2
+; CHECK-LE-IADISABLED-NEXT: movi v19.2d, #0x00ffff0000ffff
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI32_0]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_1
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x9, :lo12:.LCPI32_2]
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x8, :lo12:.LCPI32_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_4
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI32_6
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x8, :lo12:.LCPI32_4]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_5
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q3 killed $q3 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x9, :lo12:.LCPI32_6]
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x8, :lo12:.LCPI32_5]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_7
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q18, [x8, :lo12:.LCPI32_7]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI32_3
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v2.16b, v3.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q20, [x8, :lo12:.LCPI32_3]
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v0.16b, v1.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v17.16b, { v2.16b, v3.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v18.16b, { v0.16b, v1.16b }, v18.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v20.16b
+; CHECK-LE-IADISABLED-NEXT: mov v1.16b, v19.16b
+; CHECK-LE-IADISABLED-NEXT: mov v5.16b, v19.16b
+; CHECK-LE-IADISABLED-NEXT: bsl v1.16b, v16.16b, v7.16b
+; CHECK-LE-IADISABLED-NEXT: bsl v5.16b, v18.16b, v17.16b
+; CHECK-LE-IADISABLED-NEXT: bif v3.16b, v4.16b, v19.16b
+; CHECK-LE-IADISABLED-NEXT: bif v0.16b, v2.16b, v19.16b
+; CHECK-LE-IADISABLED-NEXT: stnp q5, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q3, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v16i8:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.16b, v4.16b, v5.16b, v6.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v16i8:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_0
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_6
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_6
+; CHECK-BE-IADISABLED-NEXT: adrp x10, .LCPI32_4
+; CHECK-BE-IADISABLED-NEXT: add x10, x10, :lo12:.LCPI32_4
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI32_2
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI32_2
+; CHECK-BE-IADISABLED-NEXT: ext v6.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v17.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v5.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v16.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_7
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_7
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x10]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_5
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_3
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_3
+; CHECK-BE-IADISABLED-NEXT: ld1 { v18.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI32_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI32_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v19.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v16.16b, v17.16b }, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v5.16b, v6.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v16.16b, v17.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v5.16b, v6.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v18.16b, { v5.16b, v6.16b }, v18.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v16.16b, v17.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v16.16b, v17.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v5.16b, v6.16b }, v19.16b
+; CHECK-BE-IADISABLED-NEXT: movi v6.2d, #0x00ffff0000ffff
+; CHECK-BE-IADISABLED-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v7.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v16.16b, v18.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v4.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev16 v5.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: bit v2.16b, v3.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: mov v3.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: bit v0.16b, v7.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: bit v1.16b, v16.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: bsl v3.16b, v5.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <16 x i8> %v0, <16 x i8> %v1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+ %s1 = shufflevector <16 x i8> %v2, <16 x i8> %v3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+ %shuffle = shufflevector <32 x i8> %s0, <32 x i8> %s1, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+ store <64 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v2f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, <2 x double> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2f64:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2f64:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v4.2d, v2.2d, v3.2d
+; CHECK-LE-IADISABLED-NEXT: zip2 v5.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v3.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stnp q5, q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v2f64:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.2d, v4.2d, v5.2d, v6.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v2f64:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v4.2d, v2.2d, v3.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v3.2d
+; CHECK-BE-IADISABLED-NEXT: zip2 v5.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v4.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v5.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %s1 = shufflevector <2 x double> %v2, <2 x double> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %shuffle = shufflevector <4 x double> %s0, <4 x double> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+ store <8 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v4f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4f32:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4f32:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: uzp2 v4.4s, v4.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; CHECK-LE-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v16.d[1], v5.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp q7, q4, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q16, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v4f32:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.4s, v4.4s, v5.4s, v6.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v4f32:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: uzp2 v4.4s, v4.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v6.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v5.4s, v5.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v7.4s, v16.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: mov v6.d[1], v2.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v7.d[1], v5.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s1 = shufflevector <4 x float> %v2, <4 x float> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %shuffle = shufflevector <8 x float> %s0, <8 x float> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ store <16 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store4_v8f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, <8 x half> %v3, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8f16:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-LE-IAENABLED-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8f16:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI35_5
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q3 killed $q3 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: ldr q4, [x8, :lo12:.LCPI35_1]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_7
+; CHECK-LE-IADISABLED-NEXT: ldr q5, [x9, :lo12:.LCPI35_5]
+; CHECK-LE-IADISABLED-NEXT: ldr q6, [x8, :lo12:.LCPI35_7]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_3
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IADISABLED-NEXT: adrp x9, .LCPI35_0
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x8, :lo12:.LCPI35_3]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_4
+; CHECK-LE-IADISABLED-NEXT: tbl v4.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v5.16b
+; CHECK-LE-IADISABLED-NEXT: tbl v6.16b, { v0.16b, v1.16b }, v6.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q7, [x9, :lo12:.LCPI35_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI35_4]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_6
+; CHECK-LE-IADISABLED-NEXT: ldr q16, [x8, :lo12:.LCPI35_6]
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI35_2
+; CHECK-LE-IADISABLED-NEXT: // kill: def $q2 killed $q2 killed $q2_q3 def $q2_q3
+; CHECK-LE-IADISABLED-NEXT: tbl v7.16b, { v2.16b, v3.16b }, v7.16b
+; CHECK-LE-IADISABLED-NEXT: ldr q17, [x8, :lo12:.LCPI35_2]
+; CHECK-LE-IADISABLED-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v5.4s, v5.4s
+; CHECK-LE-IADISABLED-NEXT: tbl v16.16b, { v2.16b, v3.16b }, v16.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v6.4s, v6.4s
+; CHECK-LE-IADISABLED-NEXT: tbl v2.16b, { v2.16b, v3.16b }, v17.16b
+; CHECK-LE-IADISABLED-NEXT: rev64 v3.4s, v4.4s
+; CHECK-LE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v1.4s, v5.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v4.4s, v6.4s, v16.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v3.4s, v3.4s, v7.4s
+; CHECK-LE-IADISABLED-NEXT: trn2 v0.4s, v0.4s, v2.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q4, q1, [x0, #32]
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q3, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store4_v8f16:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v3.8h, v3.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st4 { v3.8h, v4.8h, v5.8h, v6.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store4_v8f16:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI35_3
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI35_3
+; CHECK-BE-IADISABLED-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_1
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_1
+; CHECK-BE-IADISABLED-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_2
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_2
+; CHECK-BE-IADISABLED-NEXT: ext v6.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI35_7
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI35_7
+; CHECK-BE-IADISABLED-NEXT: ext v5.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v17.16b, v3.16b, v3.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v0.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI35_5
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI35_5
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI35_6
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI35_6
+; CHECK-BE-IADISABLED-NEXT: ext v16.16b, v2.16b, v2.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ld1 { v3.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: adrp x9, .LCPI35_4
+; CHECK-BE-IADISABLED-NEXT: add x9, x9, :lo12:.LCPI35_4
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v4.16b, { v5.16b, v6.16b }, v4.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v7.16b }, [x9]
+; CHECK-BE-IADISABLED-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: ld1 { v5.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI35_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI35_0
+; CHECK-BE-IADISABLED-NEXT: ld1 { v6.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v3.16b, { v16.16b, v17.16b }, v3.16b
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #48
+; CHECK-BE-IADISABLED-NEXT: rev32 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v7.16b, { v16.16b, v17.16b }, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v4.16b, v4.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v2.16b, v2.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v5.16b, { v16.16b, v17.16b }, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v6.16b, { v16.16b, v17.16b }, v6.16b
+; CHECK-BE-IADISABLED-NEXT: rev32 v3.16b, v3.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v7.16b, v7.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v4.4s, v4.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v5.16b, v5.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v6.16b, v6.16b
+; CHECK-BE-IADISABLED-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v2.4s, v2.4s, v7.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v1.4s, v1.4s, v5.4s
+; CHECK-BE-IADISABLED-NEXT: trn2 v3.4s, v4.4s, v6.4s
+; CHECK-BE-IADISABLED-NEXT: rev32 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: rev32 v2.8h, v2.8h
+; CHECK-BE-IADISABLED-NEXT: rev32 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev32 v3.8h, v3.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #32
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: st1 { v1.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v3.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %s0 = shufflevector <8 x half> %v0, <8 x half> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s1 = shufflevector <8 x half> %v2, <8 x half> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %shuffle = shufflevector <16 x half> %s0, <16 x half> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+ store <32 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count(<3 x i32> %v0, <3 x i32> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: mov d3, v2.d[1]
+; CHECK-LE-NEXT: str d0, [x0, #16]
+; CHECK-LE-NEXT: stnp d2, d3, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v2.4s
+; CHECK-BE-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-NEXT: str d1, [x0, #16]
+; CHECK-BE-NEXT: ret
+entry:
+ %shuffle = shufflevector <3 x i32> %v0, <3 x i32> %v1,
+ <6 x i32> <i32 0, i32 3, i32 1, i32 4, i32 2, i32 5>
+ store <6 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+define void @test_stnp_interleaved_store2_v2i24_non_pow2_elt_size(<2 x i24> %v0, <2 x i24> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-NEXT: fmov w9, s0
+; CHECK-LE-NEXT: str h0, [x0]
+; CHECK-LE-NEXT: mov w8, v1.s[1]
+; CHECK-LE-NEXT: dup v1.2s, v0.s[1]
+; CHECK-LE-NEXT: lsr w9, w9, #16
+; CHECK-LE-NEXT: mov w10, v2.s[1]
+; CHECK-LE-NEXT: strb w9, [x0, #2]
+; CHECK-LE-NEXT: fmov w9, s1
+; CHECK-LE-NEXT: sturh w8, [x0, #9]
+; CHECK-LE-NEXT: lsr w8, w8, #16
+; CHECK-LE-NEXT: str h1, [x0, #6]
+; CHECK-LE-NEXT: strb w8, [x0, #11]
+; CHECK-LE-NEXT: lsr w8, w10, #16
+; CHECK-LE-NEXT: lsr w9, w9, #16
+; CHECK-LE-NEXT: sturh w10, [x0, #3]
+; CHECK-LE-NEXT: strb w8, [x0, #5]
+; CHECK-LE-NEXT: strb w9, [x0, #8]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-NEXT: fmov w9, s0
+; CHECK-BE-NEXT: mov w8, v1.s[1]
+; CHECK-BE-NEXT: dup v1.2s, v0.s[1]
+; CHECK-BE-NEXT: stur b0, [x0, #2]
+; CHECK-BE-NEXT: lsr w9, w9, #8
+; CHECK-BE-NEXT: mov w10, v2.s[1]
+; CHECK-BE-NEXT: strb w8, [x0, #11]
+; CHECK-BE-NEXT: lsr w8, w8, #8
+; CHECK-BE-NEXT: strh w9, [x0]
+; CHECK-BE-NEXT: fmov w9, s1
+; CHECK-BE-NEXT: sturh w8, [x0, #9]
+; CHECK-BE-NEXT: stur b1, [x0, #8]
+; CHECK-BE-NEXT: lsr w8, w10, #8
+; CHECK-BE-NEXT: lsr w9, w9, #8
+; CHECK-BE-NEXT: strb w10, [x0, #5]
+; CHECK-BE-NEXT: sturh w8, [x0, #3]
+; CHECK-BE-NEXT: strh w9, [x0, #6]
+; CHECK-BE-NEXT: ret
+entry:
+ %shuffle = shufflevector <2 x i24> %v0, <2 x i24> %v1,
+ <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+ store <4 x i24> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+!0 = !{ i32 1 }
+
+
+define void @test_stnp_interleaved_store2_v2i32_intrinsic(<2 x i32> %v0, <2 x i32> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x i32> @llvm.vector.interleave2.v4i32(<2 x i32> %v0, <2 x i32> %v1)
+ store <4 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v4i16_intrinsic(<4 x i16> %v0, <4 x i16> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI39_0
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI39_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI39_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI39_0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <8 x i16> @llvm.vector.interleave2.v8i16(<4 x i16> %v0, <4 x i16> %v1)
+ store <8 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v8i8_intrinsic(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI40_0
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI40_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.8b, v1.8b
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8b, v0.8b
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8b, v2.8b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i8_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8b, v1.8b
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI40_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI40_0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> %v0, <8 x i8> %v1)
+ store <16 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v2f32_intrinsic(<2 x float> %v0, <2 x float> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-LE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.2s, v1.2s
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.2s, v0.2s
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %v0, <2 x float> %v1)
+ store <4 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v4f16_intrinsic(<4 x half> %v0, <4 x half> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.4h, v1.4h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-LE-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-LE-IADISABLED-NEXT: adrp x8, .LCPI42_0
+; CHECK-LE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-LE-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI42_0]
+; CHECK-LE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-LE-IADISABLED-NEXT: mov d1, v0.d[1]
+; CHECK-LE-IADISABLED-NEXT: stnp d0, d1, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v2.4h, v1.4h
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4h, v0.4h
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4h, v2.4h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4h, v1.4h
+; CHECK-BE-IADISABLED-NEXT: adrp x8, .LCPI42_0
+; CHECK-BE-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI42_0
+; CHECK-BE-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-BE-IADISABLED-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %v0, <4 x half> %v1)
+ store <8 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v2i64_intrinsic(<2 x i64> %v0, <2 x i64> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2i64_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %v0, <2 x i64> %v1)
+ store <4 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v4i32_intrinsic(<4 x i32> %v0, <4 x i32> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4i32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store2_v8i16_intrinsic(<8 x i16> %v0, <8 x i16> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8i16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
- %shuffle = shufflevector <16 x i8> %v0, <16 x i8> %v1,
- <32 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
- i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15,
- i32 8, i32 16, i32 9, i32 17, i32 10, i32 18, i32 11, i32 19,
- i32 12, i32 20, i32 13, i32 21, i32 14, i32 22, i32 15, i32 23>
- store <32 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ %interleave = call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %v0, <8 x i16> %v1)
+ store <16 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store2_v2f64(<2 x double> %v0, <2 x double> %v1, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2f64:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: zip2 v2.2d, v0.2d, v1.2d
-; CHECK-LE-NEXT: zip1 v0.2d, v0.2d, v1.2d
-; CHECK-LE-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-NEXT: ret
+
+define void @test_stnp_interleaved_store2_v16i8_intrinsic(<16 x i8> %v0, <16 x i8> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2f64:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.2d, v2.2d }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.16b, v2.16b }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v16i8_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.16b }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
- %shuffle = shufflevector <2 x double> %v0, <2 x double> %v1,
- <4 x i32> <i32 0, i32 2, i32 1, i32 3>
- store <4 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ %interleave = call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %v0, <16 x i8> %v1)
+ store <32 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store2_v4f32(<4 x float> %v0, <4 x float> %v1, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v4f32:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-NEXT: ret
+
+define void @test_stnp_interleaved_store2_v2f64_intrinsic(<2 x double> %v0, <2 x double> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v4f32:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.4s, v2.4s }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.2d, v2.2d }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v2f64_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.2d }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.2d }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
- %shuffle = shufflevector <4 x float> %v0, <4 x float> %v1,
- <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
- store <8 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %v0, <2 x double> %v1)
+ store <4 x double> %interleave, ptr %ptr, align 8, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store2_v8f16(<8 x half> %v0, <8 x half> %v1, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v8f16:
-; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: zip2 v2.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: stnp q0, q2, [x0]
-; CHECK-LE-NEXT: ret
+
+define void @test_stnp_interleaved_store2_v4f32_intrinsic(<4 x float> %v0, <4 x float> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v8f16:
-; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: st2 { v1.8h, v2.8h }, [x0]
-; CHECK-BE-NEXT: ret
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.4s, v2.4s }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v4f32_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.4s }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.4s }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
entry:
- %shuffle = shufflevector <8 x half> %v0, <8 x half> %v1,
- <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11,
- i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
- store <16 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %v0, <4 x float> %v1)
+ store <8 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2i32:
+
+define void @test_stnp_interleaved_store2_v8f16_intrinsic(<8 x half> %v0, <8 x half> %v1, ptr %ptr) {
+; CHECK-LE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-LE-IAENABLED: // %bb.0: // %entry
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-LE-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; CHECK-LE-IAENABLED-NEXT: ret
+;
+; CHECK-LE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-LE-IADISABLED: // %bb.0: // %entry
+; CHECK-LE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-IADISABLED-NEXT: stnp q0, q2, [x0]
+; CHECK-LE-IADISABLED-NEXT: ret
+;
+; CHECK-BE-IAENABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-BE-IAENABLED: // %bb.0: // %entry
+; CHECK-BE-IAENABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IAENABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IAENABLED-NEXT: ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IAENABLED-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IAENABLED-NEXT: st2 { v1.8h, v2.8h }, [x0]
+; CHECK-BE-IAENABLED-NEXT: ret
+;
+; CHECK-BE-IADISABLED-LABEL: test_stnp_interleaved_store2_v8f16_intrinsic:
+; CHECK-BE-IADISABLED: // %bb.0: // %entry
+; CHECK-BE-IADISABLED-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-IADISABLED-NEXT: add x8, x0, #16
+; CHECK-BE-IADISABLED-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-IADISABLED-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-IADISABLED-NEXT: st1 { v2.8h }, [x8]
+; CHECK-BE-IADISABLED-NEXT: st1 { v0.8h }, [x0]
+; CHECK-BE-IADISABLED-NEXT: ret
+entry:
+ %interleave = call <16 x half> @llvm.vector.interleave2.v16f16(<8 x half> %v0, <8 x half> %v1)
+ store <16 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
+ ret void
+}
+
+
+define void @test_stnp_interleaved_store3_v2i32_intrinsic(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
@@ -299,7 +2901,7 @@ define void @test_stnp_interleaved_store3_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2
; CHECK-LE-NEXT: st3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2i32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v4.2s, v2.2s
; CHECK-BE-NEXT: rev64 v3.2s, v1.2s
@@ -307,15 +2909,14 @@ define void @test_stnp_interleaved_store3_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2
; CHECK-BE-NEXT: st3 { v2.2s, v3.2s, v4.2s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x i32> %v2, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %shuffle = shufflevector <4 x i32> %s0, <4 x i32> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
- store <6 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <6 x i32> @llvm.vector.interleave3.v6i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2)
+ store <6 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4i16:
+
+define void @test_stnp_interleaved_store3_v4i16_intrinsic(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
@@ -323,7 +2924,7 @@ define void @test_stnp_interleaved_store3_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4
; CHECK-LE-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4i16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v4.4h, v2.4h
; CHECK-BE-NEXT: rev64 v3.4h, v1.4h
@@ -331,15 +2932,14 @@ define void @test_stnp_interleaved_store3_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4
; CHECK-BE-NEXT: st3 { v2.4h, v3.4h, v4.4h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x i16> %v0, <4 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i16> %v2, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <8 x i16> %s0, <8 x i16> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
- store <12 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <12 x i16> @llvm.vector.interleave3.v12i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2)
+ store <12 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8i8:
+
+define void @test_stnp_interleaved_store3_v8i8_intrinsic(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
@@ -347,7 +2947,7 @@ define void @test_stnp_interleaved_store3_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x
; CHECK-LE-NEXT: st3 { v0.8b, v1.8b, v2.8b }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8i8:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v4.8b, v2.8b
; CHECK-BE-NEXT: rev64 v3.8b, v1.8b
@@ -355,15 +2955,14 @@ define void @test_stnp_interleaved_store3_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x
; CHECK-BE-NEXT: st3 { v2.8b, v3.8b, v4.8b }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x i8> %v2, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <16 x i8> %s0, <16 x i8> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
- store <24 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ %interleave = call <24 x i8> @llvm.vector.interleave3.v24i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2)
+ store <24 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v2f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2f32:
+
+define void @test_stnp_interleaved_store3_v2f32_intrinsic(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
@@ -371,7 +2970,7 @@ define void @test_stnp_interleaved_store3_v2f32(<2 x float> %v0, <2 x float> %v1
; CHECK-LE-NEXT: st3 { v0.2s, v1.2s, v2.2s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2f32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v4.2s, v2.2s
; CHECK-BE-NEXT: rev64 v3.2s, v1.2s
@@ -379,15 +2978,14 @@ define void @test_stnp_interleaved_store3_v2f32(<2 x float> %v0, <2 x float> %v1
; CHECK-BE-NEXT: st3 { v2.2s, v3.2s, v4.2s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x float> %v0, <2 x float> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x float> %v2, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %shuffle = shufflevector <4 x float> %s0, <4 x float> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
- store <6 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <6 x float> @llvm.vector.interleave3.v6f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2)
+ store <6 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v4f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4f16:
+
+define void @test_stnp_interleaved_store3_v4f16_intrinsic(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-LE-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
@@ -395,7 +2993,7 @@ define void @test_stnp_interleaved_store3_v4f16(<4 x half> %v0, <4 x half> %v1,
; CHECK-LE-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4f16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v4.4h, v2.4h
; CHECK-BE-NEXT: rev64 v3.4h, v1.4h
@@ -403,15 +3001,14 @@ define void @test_stnp_interleaved_store3_v4f16(<4 x half> %v0, <4 x half> %v1,
; CHECK-BE-NEXT: st3 { v2.4h, v3.4h, v4.4h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x half> %v2, <4 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <8 x half> %s0, <8 x half> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
- store <12 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <12 x half> @llvm.vector.interleave3.v12f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2)
+ store <12 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2i64:
+
+define void @test_stnp_interleaved_store3_v2i64_intrinsic(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2i64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -419,7 +3016,7 @@ define void @test_stnp_interleaved_store3_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2
; CHECK-LE-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2i64:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2i64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ext v4.16b, v2.16b, v2.16b, #8
; CHECK-BE-NEXT: ext v3.16b, v1.16b, v1.16b, #8
@@ -427,15 +3024,14 @@ define void @test_stnp_interleaved_store3_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2
; CHECK-BE-NEXT: st3 { v2.2d, v3.2d, v4.2d }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x i64> %v0, <2 x i64> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x i64> %v2, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %shuffle = shufflevector <4 x i64> %s0, <4 x i64> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
- store <6 x i64> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ %interleave = call <6 x i64> @llvm.vector.interleave3.v6i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2)
+ store <6 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4i32:
+
+define void @test_stnp_interleaved_store3_v4i32_intrinsic(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -443,7 +3039,7 @@ define void @test_stnp_interleaved_store3_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4
; CHECK-LE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4i32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
@@ -454,15 +3050,14 @@ define void @test_stnp_interleaved_store3_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4
; CHECK-BE-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
- store <12 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8i16:
+
+define void @test_stnp_interleaved_store3_v8i16_intrinsic(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -470,7 +3065,7 @@ define void @test_stnp_interleaved_store3_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8
; CHECK-LE-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8i16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
@@ -481,15 +3076,14 @@ define void @test_stnp_interleaved_store3_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8
; CHECK-BE-NEXT: st3 { v2.8h, v3.8h, v4.8h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <16 x i16> %s0, <16 x i16> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
- store <24 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <24 x i16> @llvm.vector.interleave3.v24i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2)
+ store <24 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v16i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v16i8:
+
+define void @test_stnp_interleaved_store3_v16i8_intrinsic(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v16i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -497,7 +3091,7 @@ define void @test_stnp_interleaved_store3_v16i8(<16 x i8> %v0, <16 x i8> %v1, <1
; CHECK-LE-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v16i8:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v16i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
@@ -508,15 +3102,14 @@ define void @test_stnp_interleaved_store3_v16i8(<16 x i8> %v0, <16 x i8> %v1, <1
; CHECK-BE-NEXT: st3 { v2.16b, v3.16b, v4.16b }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <16 x i8> %v0, <16 x i8> %v1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
- %s1 = shufflevector <16 x i8> %v2, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <32 x i8> %s0, <32 x i8> %s1, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
- store <48 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ %interleave = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2)
+ store <48 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v2f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2f64:
+
+define void @test_stnp_interleaved_store3_v2f64_intrinsic(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v2f64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -524,7 +3117,7 @@ define void @test_stnp_interleaved_store3_v2f64(<2 x double> %v0, <2 x double> %
; CHECK-LE-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2f64:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v2f64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ext v4.16b, v2.16b, v2.16b, #8
; CHECK-BE-NEXT: ext v3.16b, v1.16b, v1.16b, #8
@@ -532,15 +3125,14 @@ define void @test_stnp_interleaved_store3_v2f64(<2 x double> %v0, <2 x double> %
; CHECK-BE-NEXT: st3 { v2.2d, v3.2d, v4.2d }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x double> %v2, <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %shuffle = shufflevector <4 x double> %s0, <4 x double> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
- store <6 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ %interleave = call <6 x double> @llvm.vector.interleave3.v6f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2)
+ store <6 x double> %interleave, ptr %ptr, align 8, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v4f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4f32:
+
+define void @test_stnp_interleaved_store3_v4f32_intrinsic(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v4f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -548,7 +3140,7 @@ define void @test_stnp_interleaved_store3_v4f32(<4 x float> %v0, <4 x float> %v1
; CHECK-LE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4f32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v4f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
@@ -559,15 +3151,14 @@ define void @test_stnp_interleaved_store3_v4f32(<4 x float> %v0, <4 x float> %v1
; CHECK-BE-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x float> %v2, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <8 x float> %s0, <8 x float> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
- store <12 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2)
+ store <12 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store3_v8f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8f16:
+
+define void @test_stnp_interleaved_store3_v8f16_intrinsic(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store3_v8f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-LE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
@@ -575,7 +3166,7 @@ define void @test_stnp_interleaved_store3_v8f16(<8 x half> %v0, <8 x half> %v1,
; CHECK-LE-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8f16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store3_v8f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
@@ -586,17 +3177,14 @@ define void @test_stnp_interleaved_store3_v8f16(<8 x half> %v0, <8 x half> %v1,
; CHECK-BE-NEXT: st3 { v2.8h, v3.8h, v4.8h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x half> %v0, <8 x half> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x half> %v2, <8 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %shuffle = shufflevector <16 x half> %s0, <16 x half> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
- store <24 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <24 x half> @llvm.vector.interleave3.v24f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2)
+ store <24 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-; Test conservative lowering of a st4 matching patterns
-define void @test_stnp_interleaved_store4_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2i32:
+define void @test_stnp_interleaved_store4_v2i32_intrinsic(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
@@ -605,7 +3193,7 @@ define void @test_stnp_interleaved_store4_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2
; CHECK-LE-NEXT: st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2i32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v6.2s, v3.2s
; CHECK-BE-NEXT: rev64 v5.2s, v2.2s
@@ -614,15 +3202,14 @@ define void @test_stnp_interleaved_store4_v2i32(<2 x i32> %v0, <2 x i32> %v1, <2
; CHECK-BE-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x i32> %v2, <2 x i32> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %shuffle = shufflevector <4 x i32> %s0, <4 x i32> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
- store <8 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3)
+ store <8 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, <4 x i16> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4i16:
+
+define void @test_stnp_interleaved_store4_v4i16_intrinsic(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, <4 x i16> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
@@ -631,7 +3218,7 @@ define void @test_stnp_interleaved_store4_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4
; CHECK-LE-NEXT: st4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4i16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v6.4h, v3.4h
; CHECK-BE-NEXT: rev64 v5.4h, v2.4h
@@ -640,15 +3227,14 @@ define void @test_stnp_interleaved_store4_v4i16(<4 x i16> %v0, <4 x i16> %v1, <4
; CHECK-BE-NEXT: st4 { v3.4h, v4.4h, v5.4h, v6.4h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x i16> %v0, <4 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i16> %v2, <4 x i16> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %shuffle = shufflevector <8 x i16> %s0, <8 x i16> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
- store <16 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <16 x i16> @llvm.vector.interleave4.v16i16(<4 x i16> %v0, <4 x i16> %v1, <4 x i16> %v2, <4 x i16> %v3)
+ store <16 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8i8:
+
+define void @test_stnp_interleaved_store4_v8i8_intrinsic(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
@@ -657,7 +3243,7 @@ define void @test_stnp_interleaved_store4_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x
; CHECK-LE-NEXT: st4 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8i8:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v6.8b, v3.8b
; CHECK-BE-NEXT: rev64 v5.8b, v2.8b
@@ -666,15 +3252,14 @@ define void @test_stnp_interleaved_store4_v8i8(<8 x i8> %v0, <8 x i8> %v1, <8 x
; CHECK-BE-NEXT: st4 { v3.8b, v4.8b, v5.8b, v6.8b }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x i8> %v2, <8 x i8> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %shuffle = shufflevector <16 x i8> %s0, <16 x i8> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
- store <32 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ %interleave = call <32 x i8> @llvm.vector.interleave4.v32i8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3)
+ store <32 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v2f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, <2 x float> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2f32:
+
+define void @test_stnp_interleaved_store4_v2f32_intrinsic(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, <2 x float> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
@@ -683,7 +3268,7 @@ define void @test_stnp_interleaved_store4_v2f32(<2 x float> %v0, <2 x float> %v1
; CHECK-LE-NEXT: st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2f32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v6.2s, v3.2s
; CHECK-BE-NEXT: rev64 v5.2s, v2.2s
@@ -692,15 +3277,14 @@ define void @test_stnp_interleaved_store4_v2f32(<2 x float> %v0, <2 x float> %v1
; CHECK-BE-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x float> %v0, <2 x float> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x float> %v2, <2 x float> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %shuffle = shufflevector <4 x float> %s0, <4 x float> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
- store <8 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <8 x float> @llvm.vector.interleave4.v8f32(<2 x float> %v0, <2 x float> %v1, <2 x float> %v2, <2 x float> %v3)
+ store <8 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v4f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, <4 x half> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4f16:
+
+define void @test_stnp_interleaved_store4_v4f16_intrinsic(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, <4 x half> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
; CHECK-LE-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
@@ -709,7 +3293,7 @@ define void @test_stnp_interleaved_store4_v4f16(<4 x half> %v0, <4 x half> %v1,
; CHECK-LE-NEXT: st4 { v0.4h, v1.4h, v2.4h, v3.4h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4f16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v6.4h, v3.4h
; CHECK-BE-NEXT: rev64 v5.4h, v2.4h
@@ -718,15 +3302,14 @@ define void @test_stnp_interleaved_store4_v4f16(<4 x half> %v0, <4 x half> %v1,
; CHECK-BE-NEXT: st4 { v3.4h, v4.4h, v5.4h, v6.4h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x half> %v2, <4 x half> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %shuffle = shufflevector <8 x half> %s0, <8 x half> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
- store <16 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <16 x half> @llvm.vector.interleave4.v16f16(<4 x half> %v0, <4 x half> %v1, <4 x half> %v2, <4 x half> %v3)
+ store <16 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2i64:
+
+define void @test_stnp_interleaved_store4_v2i64_intrinsic(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2i64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -735,7 +3318,7 @@ define void @test_stnp_interleaved_store4_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2
; CHECK-LE-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2i64:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2i64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ext v6.16b, v3.16b, v3.16b, #8
; CHECK-BE-NEXT: ext v5.16b, v2.16b, v2.16b, #8
@@ -744,15 +3327,14 @@ define void @test_stnp_interleaved_store4_v2i64(<2 x i64> %v0, <2 x i64> %v1, <2
; CHECK-BE-NEXT: st4 { v3.2d, v4.2d, v5.2d, v6.2d }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x i64> %v0, <2 x i64> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x i64> %v2, <2 x i64> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %shuffle = shufflevector <4 x i64> %s0, <4 x i64> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
- store <8 x i64> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ %interleave = call <8 x i64> @llvm.vector.interleave4.v8i64(<2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3)
+ store <8 x i64> %interleave, ptr %ptr, align 8, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4i32:
+
+define void @test_stnp_interleaved_store4_v4i32_intrinsic(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4i32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -761,7 +3343,7 @@ define void @test_stnp_interleaved_store4_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4
; CHECK-LE-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4i32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4i32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
@@ -774,15 +3356,14 @@ define void @test_stnp_interleaved_store4_v4i32(<4 x i32> %v0, <4 x i32> %v1, <4
; CHECK-BE-NEXT: st4 { v3.4s, v4.4s, v5.4s, v6.4s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %shuffle = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
- store <16 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
+ store <16 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8i16:
+
+define void @test_stnp_interleaved_store4_v8i16_intrinsic(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8i16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -791,7 +3372,7 @@ define void @test_stnp_interleaved_store4_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8
; CHECK-LE-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8i16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8i16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v3.8h, v3.8h
; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
@@ -804,15 +3385,14 @@ define void @test_stnp_interleaved_store4_v8i16(<8 x i16> %v0, <8 x i16> %v1, <8
; CHECK-BE-NEXT: st4 { v3.8h, v4.8h, v5.8h, v6.8h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x i16> %v2, <8 x i16> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %shuffle = shufflevector <16 x i16> %s0, <16 x i16> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
- store <32 x i16> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> %v0, <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %v3)
+ store <32 x i16> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v16i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v16i8:
+
+define void @test_stnp_interleaved_store4_v16i8_intrinsic(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v16i8_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -821,7 +3401,7 @@ define void @test_stnp_interleaved_store4_v16i8(<16 x i8> %v0, <16 x i8> %v1, <1
; CHECK-LE-NEXT: st4 { v0.16b, v1.16b, v2.16b, v3.16b }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v16i8:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v16i8_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v3.16b, v3.16b
; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
@@ -834,15 +3414,14 @@ define void @test_stnp_interleaved_store4_v16i8(<16 x i8> %v0, <16 x i8> %v1, <1
; CHECK-BE-NEXT: st4 { v3.16b, v4.16b, v5.16b, v6.16b }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <16 x i8> %v0, <16 x i8> %v1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
- %s1 = shufflevector <16 x i8> %v2, <16 x i8> %v3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
- %shuffle = shufflevector <32 x i8> %s0, <32 x i8> %s1, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
- store <64 x i8> %shuffle, ptr %ptr, align 1, !nontemporal !0
+ %interleave = call <64 x i8> @llvm.vector.interleave4.v64i8(<16 x i8> %v0, <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %v3)
+ store <64 x i8> %interleave, ptr %ptr, align 1, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v2f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, <2 x double> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2f64:
+
+define void @test_stnp_interleaved_store4_v2f64_intrinsic(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, <2 x double> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v2f64_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -851,7 +3430,7 @@ define void @test_stnp_interleaved_store4_v2f64(<2 x double> %v0, <2 x double> %
; CHECK-LE-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2f64:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v2f64_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ext v6.16b, v3.16b, v3.16b, #8
; CHECK-BE-NEXT: ext v5.16b, v2.16b, v2.16b, #8
@@ -860,15 +3439,14 @@ define void @test_stnp_interleaved_store4_v2f64(<2 x double> %v0, <2 x double> %
; CHECK-BE-NEXT: st4 { v3.2d, v4.2d, v5.2d, v6.2d }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x double> %v2, <2 x double> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %shuffle = shufflevector <4 x double> %s0, <4 x double> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
- store <8 x double> %shuffle, ptr %ptr, align 8, !nontemporal !0
+ %interleave = call <8 x double> @llvm.vector.interleave4.v8f64(<2 x double> %v0, <2 x double> %v1, <2 x double> %v2, <2 x double> %v3)
+ store <8 x double> %interleave, ptr %ptr, align 8, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v4f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4f32:
+
+define void @test_stnp_interleaved_store4_v4f32_intrinsic(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v4f32_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -877,7 +3455,7 @@ define void @test_stnp_interleaved_store4_v4f32(<4 x float> %v0, <4 x float> %v1
; CHECK-LE-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4f32:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v4f32_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
@@ -890,15 +3468,14 @@ define void @test_stnp_interleaved_store4_v4f32(<4 x float> %v0, <4 x float> %v1
; CHECK-BE-NEXT: st4 { v3.4s, v4.4s, v5.4s, v6.4s }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x float> %v2, <4 x float> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %shuffle = shufflevector <8 x float> %s0, <8 x float> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
- store <16 x float> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <16 x float> @llvm.vector.interleave4.v16f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3)
+ store <16 x float> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store4_v8f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, <8 x half> %v3, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8f16:
+
+define void @test_stnp_interleaved_store4_v8f16_intrinsic(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, <8 x half> %v3, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store4_v8f16_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
; CHECK-LE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
@@ -907,7 +3484,7 @@ define void @test_stnp_interleaved_store4_v8f16(<8 x half> %v0, <8 x half> %v1,
; CHECK-LE-NEXT: st4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8f16:
+; CHECK-BE-LABEL: test_stnp_interleaved_store4_v8f16_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v3.8h, v3.8h
; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
@@ -920,15 +3497,14 @@ define void @test_stnp_interleaved_store4_v8f16(<8 x half> %v0, <8 x half> %v1,
; CHECK-BE-NEXT: st4 { v3.8h, v4.8h, v5.8h, v6.8h }, [x0]
; CHECK-BE-NEXT: ret
entry:
- %s0 = shufflevector <8 x half> %v0, <8 x half> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x half> %v2, <8 x half> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %shuffle = shufflevector <16 x half> %s0, <16 x half> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
- store <32 x half> %shuffle, ptr %ptr, align 2, !nontemporal !0
+ %interleave = call <32 x half> @llvm.vector.interleave4.v32f16(<8 x half> %v0, <8 x half> %v1, <8 x half> %v2, <8 x half> %v3)
+ store <32 x half> %interleave, ptr %ptr, align 2, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count(<3 x i32> %v0, <3 x i32> %v1, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count:
+
+define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic(<3 x i32> %v0, <3 x i32> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: zip1 v2.4s, v0.4s, v1.4s
; CHECK-LE-NEXT: zip2 v0.4s, v0.4s, v1.4s
@@ -937,7 +3513,7 @@ define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count(<3 x i32> %v0
; CHECK-LE-NEXT: stnp d2, d3, [x0]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count:
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v3i32_non_pow2_elt_count_intrinsic:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
@@ -950,65 +3526,66 @@ define void @test_stnp_interleaved_store2_v3i32_non_pow2_elt_count(<3 x i32> %v0
; CHECK-BE-NEXT: str d1, [x0, #16]
; CHECK-BE-NEXT: ret
entry:
- %shuffle = shufflevector <3 x i32> %v0, <3 x i32> %v1,
- <6 x i32> <i32 0, i32 3, i32 1, i32 4, i32 2, i32 5>
- store <6 x i32> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <6 x i32> @llvm.vector.interleave2.v6i32(<3 x i32> %v0, <3 x i32> %v1)
+ store <6 x i32> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-define void @test_stnp_interleaved_store2_v2i24_non_pow2_elt_size(<2 x i24> %v0, <2 x i24> %v1, ptr %ptr) {
-; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size:
+
+define void @test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic(<2 x i24> %v0, <2 x i24> %v1, ptr %ptr) {
+; CHECK-LE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
; CHECK-LE-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-LE-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-LE-NEXT: mov v2.16b, v0.16b
+; CHECK-LE-NEXT: // kill: def $d1 killed $d1 def $q1
; CHECK-LE-NEXT: fmov w9, s0
; CHECK-LE-NEXT: str h0, [x0]
; CHECK-LE-NEXT: mov w8, v1.s[1]
-; CHECK-LE-NEXT: dup v1.2s, v0.s[1]
+; CHECK-LE-NEXT: mov v2.d[1], v1.d[0]
; CHECK-LE-NEXT: lsr w9, w9, #16
-; CHECK-LE-NEXT: mov w10, v2.s[1]
; CHECK-LE-NEXT: strb w9, [x0, #2]
-; CHECK-LE-NEXT: fmov w9, s1
+; CHECK-LE-NEXT: lsr w9, w8, #16
; CHECK-LE-NEXT: sturh w8, [x0, #9]
-; CHECK-LE-NEXT: lsr w8, w8, #16
-; CHECK-LE-NEXT: str h1, [x0, #6]
-; CHECK-LE-NEXT: strb w8, [x0, #11]
+; CHECK-LE-NEXT: rev64 v3.4s, v2.4s
+; CHECK-LE-NEXT: strb w9, [x0, #11]
+; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-LE-NEXT: mov w10, v2.s[2]
+; CHECK-LE-NEXT: mov w11, v2.s[1]
; CHECK-LE-NEXT: lsr w8, w10, #16
-; CHECK-LE-NEXT: lsr w9, w9, #16
-; CHECK-LE-NEXT: sturh w10, [x0, #3]
-; CHECK-LE-NEXT: strb w8, [x0, #5]
-; CHECK-LE-NEXT: strb w9, [x0, #8]
+; CHECK-LE-NEXT: lsr w9, w11, #16
+; CHECK-LE-NEXT: strh w10, [x0, #6]
+; CHECK-LE-NEXT: sturh w11, [x0, #3]
+; CHECK-LE-NEXT: strb w8, [x0, #8]
+; CHECK-LE-NEXT: strb w9, [x0, #5]
; CHECK-LE-NEXT: ret
;
-; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size:
+; CHECK-BE-LABEL: test_stnp_interleaved_store2_v2i24_non_pow2_elt_size_intrinsic:
; CHECK-BE: // %bb.0: // %entry
-; CHECK-BE-NEXT: rev64 v1.2s, v1.2s
; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-NEXT: zip1 v2.2s, v0.2s, v1.2s
+; CHECK-BE-NEXT: rev64 v1.2s, v1.2s
+; CHECK-BE-NEXT: mov v2.16b, v0.16b
; CHECK-BE-NEXT: fmov w9, s0
; CHECK-BE-NEXT: mov w8, v1.s[1]
-; CHECK-BE-NEXT: dup v1.2s, v0.s[1]
; CHECK-BE-NEXT: stur b0, [x0, #2]
+; CHECK-BE-NEXT: mov v2.d[1], v1.d[0]
; CHECK-BE-NEXT: lsr w9, w9, #8
-; CHECK-BE-NEXT: mov w10, v2.s[1]
; CHECK-BE-NEXT: strb w8, [x0, #11]
-; CHECK-BE-NEXT: lsr w8, w8, #8
; CHECK-BE-NEXT: strh w9, [x0]
-; CHECK-BE-NEXT: fmov w9, s1
-; CHECK-BE-NEXT: sturh w8, [x0, #9]
-; CHECK-BE-NEXT: stur b1, [x0, #8]
+; CHECK-BE-NEXT: lsr w9, w8, #8
+; CHECK-BE-NEXT: rev64 v3.4s, v2.4s
+; CHECK-BE-NEXT: sturh w9, [x0, #9]
+; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-BE-NEXT: mov w10, v2.s[2]
+; CHECK-BE-NEXT: mov w11, v2.s[1]
; CHECK-BE-NEXT: lsr w8, w10, #8
-; CHECK-BE-NEXT: lsr w9, w9, #8
-; CHECK-BE-NEXT: strb w10, [x0, #5]
-; CHECK-BE-NEXT: sturh w8, [x0, #3]
-; CHECK-BE-NEXT: strh w9, [x0, #6]
+; CHECK-BE-NEXT: lsr w9, w11, #8
+; CHECK-BE-NEXT: strb w10, [x0, #8]
+; CHECK-BE-NEXT: strb w11, [x0, #5]
+; CHECK-BE-NEXT: strh w8, [x0, #6]
+; CHECK-BE-NEXT: sturh w9, [x0, #3]
; CHECK-BE-NEXT: ret
entry:
- %shuffle = shufflevector <2 x i24> %v0, <2 x i24> %v1,
- <4 x i32> <i32 0, i32 2, i32 1, i32 3>
- store <4 x i24> %shuffle, ptr %ptr, align 4, !nontemporal !0
+ %interleave = call <4 x i24> @llvm.vector.interleave2.v4i24(<2 x i24> %v0, <2 x i24> %v1)
+ store <4 x i24> %interleave, ptr %ptr, align 4, !nontemporal !0
ret void
}
-
-!0 = !{ i32 1 }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-ld2-alloca.ll b/llvm/test/CodeGen/AArch64/sve-fixed-ld2-alloca.ll
index 6d3612c909032..73caeebf2a192 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-ld2-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-ld2-alloca.ll
@@ -1,12 +1,56 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target triple = "aarch64-unknown-linux-gnu"
declare void @def(ptr)
define void @st1d_fixed(ptr %ptr) #0 {
-; CHECK-LABEL: st1d_fixed:
+; CHECK-IAENABLED-LABEL: st1d_fixed:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: sub sp, sp, #160
+; CHECK-IAENABLED-NEXT: stp x20, x19, [sp, #144] // 16-byte Folded Spill
+; CHECK-IAENABLED-NEXT: mov x19, x0
+; CHECK-IAENABLED-NEXT: mov x0, sp
+; CHECK-IAENABLED-NEXT: str x30, [sp, #128] // 8-byte Spill
+; CHECK-IAENABLED-NEXT: mov x20, sp
+; CHECK-IAENABLED-NEXT: bl def
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x20]
+; CHECK-IAENABLED-NEXT: ldr x30, [sp, #128] // 8-byte Reload
+; CHECK-IAENABLED-NEXT: str z0, [x19]
+; CHECK-IAENABLED-NEXT: ldp x20, x19, [sp, #144] // 16-byte Folded Reload
+; CHECK-IAENABLED-NEXT: add sp, sp, #160
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: st1d_fixed:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #160
+; CHECK-IADISABLED-NEXT: stp x20, x19, [sp, #144] // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: mov x19, x0
+; CHECK-IADISABLED-NEXT: mov x0, sp
+; CHECK-IADISABLED-NEXT: str x30, [sp, #128] // 8-byte Spill
+; CHECK-IADISABLED-NEXT: mov x20, sp
+; CHECK-IADISABLED-NEXT: bl def
+; CHECK-IADISABLED-NEXT: ldr z0, [x20, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x20]
+; CHECK-IADISABLED-NEXT: ldr x30, [sp, #128] // 8-byte Reload
+; CHECK-IADISABLED-NEXT: uzp1 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: str z0, [x19]
+; CHECK-IADISABLED-NEXT: ldp x20, x19, [sp, #144] // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: add sp, sp, #160
+; CHECK-IADISABLED-NEXT: ret
+ %alloc = alloca [16 x double]
+ call void @def(ptr %alloc)
+ %load = load <8 x double>, ptr %alloc
+ %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ store <4 x double> %strided.vec, ptr %ptr
+ ret void
+}
+
+define void @st1d_fixed_intrinsic(ptr %ptr) #0 {
+; CHECK-LABEL: st1d_fixed_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #160
; CHECK-NEXT: stp x20, x19, [sp, #144] // 16-byte Folded Spill
@@ -15,9 +59,10 @@ define void @st1d_fixed(ptr %ptr) #0 {
; CHECK-NEXT: str x30, [sp, #128] // 8-byte Spill
; CHECK-NEXT: mov x20, sp
; CHECK-NEXT: bl def
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x20]
+; CHECK-NEXT: ldr z0, [x20, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x20]
; CHECK-NEXT: ldr x30, [sp, #128] // 8-byte Reload
+; CHECK-NEXT: uzp1 z0.d, z1.d, z0.d
; CHECK-NEXT: str z0, [x19]
; CHECK-NEXT: ldp x20, x19, [sp, #144] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #160
@@ -25,7 +70,8 @@ define void @st1d_fixed(ptr %ptr) #0 {
%alloc = alloca [16 x double]
call void @def(ptr %alloc)
%load = load <8 x double>, ptr %alloc
- %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %deinterleaved = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> %load)
+ %strided.vec = extractvalue { <4 x double>, <4 x double> } %deinterleaved, 0
store <4 x double> %strided.vec, ptr %ptr
ret void
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index 59329624e75bb..4300bcf317d12 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target triple = "aarch64-unknown-linux-gnu"
@@ -8,14 +9,21 @@ target triple = "aarch64-unknown-linux-gnu"
; hang when expanding BUILD_VECTOR. Here we just validate the promblematic case
; successfully exits code generation.
define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) vscale_range(2,2) #0 {
-; CHECK-LABEL: hang_when_merging_stores_after_legalisation:
-; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov z0.s, s0
-; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: ext z1.b, z1.b, z0.b, #16
-; CHECK-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: hang_when_merging_stores_after_legalisation:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-IAENABLED-NEXT: mov z0.s, s0
+; CHECK-IAENABLED-NEXT: movprfx z1, z0
+; CHECK-IAENABLED-NEXT: ext z1.b, z1.b, z0.b, #16
+; CHECK-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: hang_when_merging_stores_after_legalisation:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-IADISABLED-NEXT: mov z0.s, s0
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%splat = shufflevector <2 x i32> %b, <2 x i32> poison, <8 x i32> zeroinitializer
%interleaved.vec = shufflevector <8 x i32> %splat, <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, ptr %a, align 4
@@ -40,4 +48,31 @@ exit:
ret void
}
+
+define void @hang_when_merging_stores_after_legalisation_intrinsic(ptr %a, <2 x i32> %b) vscale_range(2,2) #0 {
+; CHECK-IAENABLED-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-IAENABLED-NEXT: dup v0.4s, v0.s[0]
+; CHECK-IAENABLED-NEXT: mov v1.16b, v0.16b
+; CHECK-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI2_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-IADISABLED-NEXT: dup v0.4s, v0.s[0]
+; CHECK-IADISABLED-NEXT: ldr z1, [x8]
+; CHECK-IADISABLED-NEXT: mov z0.q, q0
+; CHECK-IADISABLED-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
+ %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <4 x i32> zeroinitializer
+ %interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %splat, <4 x i32> %splat)
+ store <8 x i32> %interleaved.vec, ptr %a, align 4
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
index 1306331d2e44a..133bac768d59e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ld2-alloca.ll
@@ -1,30 +1,51 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
+; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefixes=NONEON-NOSVE,NONEON-NOSVE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefixes=NONEON-NOSVE,NONEON-NOSVE-IADISABLED
target triple = "aarch64-unknown-linux-gnu"
declare void @def(ptr)
define void @alloc_v4i8(ptr %st_ptr) nounwind {
-; CHECK-LABEL: alloc_v4i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT: mov x19, x0
-; CHECK-NEXT: add x0, sp, #12
-; CHECK-NEXT: add x20, sp, #12
-; CHECK-NEXT: bl def
-; CHECK-NEXT: ptrue p0.b, vl2
-; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x20]
-; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: mov z1.b, z0.b[1]
-; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: st1b { z0.s }, p0, [x19]
-; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: alloc_v4i8:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-IAENABLED-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-IAENABLED-NEXT: mov x19, x0
+; CHECK-IAENABLED-NEXT: add x0, sp, #12
+; CHECK-IAENABLED-NEXT: add x20, sp, #12
+; CHECK-IAENABLED-NEXT: bl def
+; CHECK-IAENABLED-NEXT: ptrue p0.b, vl2
+; CHECK-IAENABLED-NEXT: ld2b { z0.b, z1.b }, p0/z, [x20]
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl2
+; CHECK-IAENABLED-NEXT: mov z1.b, z0.b[1]
+; CHECK-IAENABLED-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-IAENABLED-NEXT: st1b { z0.s }, p0, [x19]
+; CHECK-IAENABLED-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-IAENABLED-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: alloc_v4i8:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-IADISABLED-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: mov x19, x0
+; CHECK-IADISABLED-NEXT: add x0, sp, #12
+; CHECK-IADISABLED-NEXT: add x20, sp, #12
+; CHECK-IADISABLED-NEXT: bl def
+; CHECK-IADISABLED-NEXT: ptrue p0.h, vl4
+; CHECK-IADISABLED-NEXT: ld1b { z0.h }, p0/z, [x20]
+; CHECK-IADISABLED-NEXT: ptrue p0.s, vl2
+; CHECK-IADISABLED-NEXT: mov z1.h, z0.h[2]
+; CHECK-IADISABLED-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-IADISABLED-NEXT: st1b { z0.s }, p0, [x19]
+; CHECK-IADISABLED-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
;
; NONEON-NOSVE-LABEL: alloc_v4i8:
; NONEON-NOSVE: // %bb.0:
@@ -166,26 +187,138 @@ define void @alloc_v32i8(ptr %st_ptr) nounwind {
define void @alloc_v8f64(ptr %st_ptr) nounwind {
-; CHECK-LABEL: alloc_v8f64:
+; CHECK-IAENABLED-LABEL: alloc_v8f64:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: sub sp, sp, #96
+; CHECK-IAENABLED-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-IAENABLED-NEXT: mov x19, x0
+; CHECK-IAENABLED-NEXT: mov x0, sp
+; CHECK-IAENABLED-NEXT: str x30, [sp, #64] // 8-byte Spill
+; CHECK-IAENABLED-NEXT: mov x20, sp
+; CHECK-IAENABLED-NEXT: bl def
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: mov x8, #4 // =0x4
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x20]
+; CHECK-IAENABLED-NEXT: ld2d { z1.d, z2.d }, p0/z, [x20, x8, lsl #3]
+; CHECK-IAENABLED-NEXT: ldr x30, [sp, #64] // 8-byte Reload
+; CHECK-IAENABLED-NEXT: stp q0, q1, [x19]
+; CHECK-IAENABLED-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-IAENABLED-NEXT: add sp, sp, #96
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: alloc_v8f64:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: sub sp, sp, #80
+; CHECK-IADISABLED-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: mov x19, x0
+; CHECK-IADISABLED-NEXT: mov x0, sp
+; CHECK-IADISABLED-NEXT: bl def
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [sp]
+; CHECK-IADISABLED-NEXT: zip1 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x19]
+; CHECK-IADISABLED-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: add sp, sp, #80
+; CHECK-IADISABLED-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: alloc_v8f64:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #176
+; NONEON-NOSVE-NEXT: stp x30, x19, [sp, #160] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT: mov x19, x0
+; NONEON-NOSVE-NEXT: add x0, sp, #96
+; NONEON-NOSVE-NEXT: bl def
+; NONEON-NOSVE-NEXT: ldp q1, q0, [sp, #96]
+; NONEON-NOSVE-NEXT: ldp q2, q3, [sp, #128]
+; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #48]
+; NONEON-NOSVE-NEXT: ldr d1, [sp, #64]
+; NONEON-NOSVE-NEXT: ldr d0, [sp, #48]
+; NONEON-NOSVE-NEXT: stp q2, q3, [sp]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT: ldr d1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr d0, [sp]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT: ldr q1, [sp, #80]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q1, q0, [x19]
+; NONEON-NOSVE-NEXT: ldp x30, x19, [sp, #160] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: add sp, sp, #176
+; NONEON-NOSVE-NEXT: ret
+ %alloc = alloca [8 x double]
+ call void @def(ptr %alloc)
+ %load = load <8 x double>, ptr %alloc
+ %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ store <4 x double> %strided.vec, ptr %st_ptr
+ ret void
+}
+
+
+define void @alloc_v4i8_intrinsic(ptr %st_ptr) nounwind {
+; CHECK-LABEL: alloc_v4i8_intrinsic:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #96
-; CHECK-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT: mov x19, x0
+; CHECK-NEXT: add x0, sp, #12
+; CHECK-NEXT: add x20, sp, #12
+; CHECK-NEXT: bl def
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: ld1b { z0.h }, p0/z, [x20]
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: mov z1.h, z0.h[2]
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: mov z0.s, w8
+; CHECK-NEXT: mov z1.s, w9
+; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: st1b { z0.s }, p0, [x19]
+; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: alloc_v4i8_intrinsic:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; NONEON-NOSVE-NEXT: mov x19, x0
+; NONEON-NOSVE-NEXT: add x0, sp, #12
+; NONEON-NOSVE-NEXT: bl def
+; NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; NONEON-NOSVE-NEXT: ldrb w9, [sp, #12]
+; NONEON-NOSVE-NEXT: strb w8, [x19, #1]
+; NONEON-NOSVE-NEXT: strb w9, [x19]
+; NONEON-NOSVE-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: ret
+ %alloc = alloca [4 x i8]
+ call void @def(ptr %alloc)
+ %load = load <4 x i8>, ptr %alloc
+ %deinterleaved = call { <2 x i8>, <2 x i8> } @llvm.vector.deinterleave2.v4i8(<4 x i8> %load)
+ %strided.vec = extractvalue { <2 x i8>, <2 x i8> } %deinterleaved, 0
+ store <2 x i8> %strided.vec, ptr %st_ptr
+ ret void
+}
+
+
+define void @alloc_v8f64_intrinsic(ptr %st_ptr) nounwind {
+; CHECK-LABEL: alloc_v8f64_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: mov x19, x0
; CHECK-NEXT: mov x0, sp
-; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
-; CHECK-NEXT: mov x20, sp
; CHECK-NEXT: bl def
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: mov x8, #4 // =0x4
-; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x20]
-; CHECK-NEXT: ld2d { z1.d, z2.d }, p0/z, [x20, x8, lsl #3]
-; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
-; CHECK-NEXT: stp q0, q1, [x19]
-; CHECK-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; CHECK-NEXT: add sp, sp, #96
+; CHECK-NEXT: ldp q1, q0, [sp, #32]
+; CHECK-NEXT: ldp q3, q2, [sp]
+; CHECK-NEXT: zip1 z0.d, z1.d, z0.d
+; CHECK-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-NEXT: stp q1, q0, [x19]
+; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: ret
;
-; NONEON-NOSVE-LABEL: alloc_v8f64:
+; NONEON-NOSVE-LABEL: alloc_v8f64_intrinsic:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: sub sp, sp, #176
; NONEON-NOSVE-NEXT: stp x30, x19, [sp, #160] // 16-byte Folded Spill
@@ -211,7 +344,11 @@ define void @alloc_v8f64(ptr %st_ptr) nounwind {
%alloc = alloca [8 x double]
call void @def(ptr %alloc)
%load = load <8 x double>, ptr %alloc
- %strided.vec = shufflevector <8 x double> %load, <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %deinterleaved = call { <4 x double>, <4 x double> } @llvm.vector.deinterleave2.v8f64(<8 x double> %load)
+ %strided.vec = extractvalue { <4 x double>, <4 x double> } %deinterleaved, 0
store <4 x double> %strided.vec, ptr %st_ptr
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NONEON-NOSVE-IADISABLED: {{.*}}
+; NONEON-NOSVE-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
index c5e6c05f17807..986b776a9a586 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll
@@ -1,19 +1,28 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
+; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefixes=NONEON-NOSVE,NONEON-NOSVE-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefixes=NONEON-NOSVE,NONEON-NOSVE-IADISABLED
target triple = "aarch64-unknown-linux-gnu"
define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) {
-; CHECK-LABEL: hang_when_merging_stores_after_legalisation:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov z0.s, s0
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: mov z1.d, z0.d
-; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: hang_when_merging_stores_after_legalisation:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov z0.s, s0
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl4
+; CHECK-IAENABLED-NEXT: mov z1.d, z0.d
+; CHECK-IAENABLED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: hang_when_merging_stores_after_legalisation:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: mov z0.s, s0
+; CHECK-IADISABLED-NEXT: stp q0, q0, [x0]
+; CHECK-IADISABLED-NEXT: ret
;
; NONEON-NOSVE-LABEL: hang_when_merging_stores_after_legalisation:
; NONEON-NOSVE: // %bb.0:
@@ -34,11 +43,24 @@ define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) {
}
define void @interleave_store_without_splat(ptr %a, <4 x i32> %v1, <4 x i32> %v2) {
-; CHECK-LABEL: interleave_store_without_splat:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: interleave_store_without_splat:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl4
+; CHECK-IAENABLED-NEXT: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: interleave_store_without_splat:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: mov z2.s, z1.s[3]
+; CHECK-IADISABLED-NEXT: mov z3.s, z0.s[3]
+; CHECK-IADISABLED-NEXT: mov z4.s, z1.s[2]
+; CHECK-IADISABLED-NEXT: mov z5.s, z0.s[2]
+; CHECK-IADISABLED-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-IADISABLED-NEXT: zip1 z2.s, z3.s, z2.s
+; CHECK-IADISABLED-NEXT: zip1 z3.s, z5.s, z4.s
+; CHECK-IADISABLED-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ret
;
; NONEON-NOSVE-LABEL: interleave_store_without_splat:
; NONEON-NOSVE: // %bb.0:
@@ -69,16 +91,38 @@ define void @interleave_store_without_splat(ptr %a, <4 x i32> %v1, <4 x i32> %v2
}
define void @interleave_store_legalization(ptr %a, <8 x i32> %v1, <8 x i32> %v2) {
-; CHECK-LABEL: interleave_store_legalization:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov z5.d, z2.d
-; CHECK-NEXT: mov z4.d, z0.d
-; CHECK-NEXT: mov x8, #8 // =0x8
-; CHECK-NEXT: mov z2.d, z3.d
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: st2w { z4.s, z5.s }, p0, [x0]
-; CHECK-NEXT: st2w { z1.s, z2.s }, p0, [x0, x8, lsl #2]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: interleave_store_legalization:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov z5.d, z2.d
+; CHECK-IAENABLED-NEXT: mov z4.d, z0.d
+; CHECK-IAENABLED-NEXT: mov x8, #8 // =0x8
+; CHECK-IAENABLED-NEXT: mov z2.d, z3.d
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl4
+; CHECK-IAENABLED-NEXT: st2w { z4.s, z5.s }, p0, [x0]
+; CHECK-IAENABLED-NEXT: st2w { z1.s, z2.s }, p0, [x0, x8, lsl #2]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: interleave_store_legalization:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: mov z4.s, z3.s[3]
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[3]
+; CHECK-IADISABLED-NEXT: mov z6.s, z3.s[2]
+; CHECK-IADISABLED-NEXT: mov z7.s, z1.s[2]
+; CHECK-IADISABLED-NEXT: mov z16.s, z2.s[3]
+; CHECK-IADISABLED-NEXT: mov z17.s, z0.s[3]
+; CHECK-IADISABLED-NEXT: mov z18.s, z2.s[2]
+; CHECK-IADISABLED-NEXT: mov z19.s, z0.s[2]
+; CHECK-IADISABLED-NEXT: zip1 z1.s, z1.s, z3.s
+; CHECK-IADISABLED-NEXT: zip1 z4.s, z5.s, z4.s
+; CHECK-IADISABLED-NEXT: zip1 z0.s, z0.s, z2.s
+; CHECK-IADISABLED-NEXT: zip1 z5.s, z7.s, z6.s
+; CHECK-IADISABLED-NEXT: zip1 z3.s, z17.s, z16.s
+; CHECK-IADISABLED-NEXT: zip1 z6.s, z19.s, z18.s
+; CHECK-IADISABLED-NEXT: zip1 z4.d, z5.d, z4.d
+; CHECK-IADISABLED-NEXT: zip1 z2.d, z6.d, z3.d
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q2, [x0]
+; CHECK-IADISABLED-NEXT: ret
;
; NONEON-NOSVE-LABEL: interleave_store_legalization:
; NONEON-NOSVE: // %bb.0:
@@ -144,3 +188,140 @@ vector.body:
exit:
ret void
}
+
+
+define void @hang_when_merging_stores_after_legalisation_intrinsic(ptr %a, <2 x i32> %b) {
+; CHECK-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, s0
+; CHECK-NEXT: stp q0, q0, [x0]
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: hang_when_merging_stores_after_legalisation_intrinsic:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; NONEON-NOSVE-NEXT: stp w8, w8, [sp, #24]
+; NONEON-NOSVE-NEXT: stp w8, w8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: stp q0, q0, [x0]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: ret
+ %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <4 x i32> zeroinitializer
+ %interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %splat, <4 x i32> %splat)
+ store <8 x i32> %interleaved.vec, ptr %a, align 4
+ ret void
+}
+
+
+define void @interleave_store_without_splat_intrinsic(ptr %a, <4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: interleave_store_without_splat_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z2.s, z1.s[3]
+; CHECK-NEXT: mov z3.s, z0.s[3]
+; CHECK-NEXT: mov z4.s, z1.s[2]
+; CHECK-NEXT: mov z5.s, z0.s[2]
+; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-NEXT: zip1 z2.s, z3.s, z2.s
+; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
+; CHECK-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-NEXT: stp q0, q1, [x0]
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: interleave_store_without_splat_intrinsic:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #64
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #36]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #32]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #44]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; NONEON-NOSVE-NEXT: ldr q1, [sp]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #40]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]
+; NONEON-NOSVE-NEXT: stp q1, q0, [x0]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: ret
+ %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v1, <4 x i32> %v2)
+ store <8 x i32> %interleaved, ptr %a, align 1
+ ret void
+}
+
+
+define void @interleave_store_legalization_intrinsic(ptr %a, <8 x i32> %v1, <8 x i32> %v2) {
+; CHECK-LABEL: interleave_store_legalization_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z4.s, z3.s[3]
+; CHECK-NEXT: mov z5.s, z1.s[3]
+; CHECK-NEXT: mov z6.s, z3.s[2]
+; CHECK-NEXT: mov z7.s, z1.s[2]
+; CHECK-NEXT: mov z16.s, z2.s[3]
+; CHECK-NEXT: mov z17.s, z0.s[3]
+; CHECK-NEXT: mov z18.s, z2.s[2]
+; CHECK-NEXT: mov z19.s, z0.s[2]
+; CHECK-NEXT: zip1 z1.s, z1.s, z3.s
+; CHECK-NEXT: zip1 z4.s, z5.s, z4.s
+; CHECK-NEXT: zip1 z0.s, z0.s, z2.s
+; CHECK-NEXT: zip1 z5.s, z7.s, z6.s
+; CHECK-NEXT: zip1 z3.s, z17.s, z16.s
+; CHECK-NEXT: zip1 z6.s, z19.s, z18.s
+; CHECK-NEXT: zip1 z4.d, z5.d, z4.d
+; CHECK-NEXT: zip1 z2.d, z6.d, z3.d
+; CHECK-NEXT: stp q1, q4, [x0, #32]
+; CHECK-NEXT: stp q0, q2, [x0]
+; CHECK-NEXT: ret
+;
+; NONEON-NOSVE-LABEL: interleave_store_legalization_intrinsic:
+; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #128
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128
+; NONEON-NOSVE-NEXT: stp q1, q3, [sp, #16]
+; NONEON-NOSVE-NEXT: stp q0, q2, [sp, #80]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #100]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #84]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #72]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #96]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #80]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #64]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #108]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #92]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #120]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #104]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #88]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #112]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #36]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; NONEON-NOSVE-NEXT: ldr q3, [sp, #112]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #32]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #44]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; NONEON-NOSVE-NEXT: ldr q1, [sp]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT: ldr w9, [sp, #40]
+; NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp q0, q2, [sp, #48]
+; NONEON-NOSVE-NEXT: stp q2, q3, [x0]
+; NONEON-NOSVE-NEXT: stp q1, q0, [x0, #32]
+; NONEON-NOSVE-NEXT: add sp, sp, #128
+; NONEON-NOSVE-NEXT: ret
+ %interleaved.vec = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v1, <8 x i32> %v2)
+ store <16 x i32> %interleaved.vec, ptr %a, align 4
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NONEON-NOSVE-IADISABLED: {{.*}}
+; NONEON-NOSVE-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 25bceea3fe2b8..4870243e0e4a4 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
; CHECK-LABEL: loop1:
@@ -142,81 +143,159 @@ for.body: ; preds = %for.body.preheader2
}
define void @loop2(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
-; CHECK-LABEL: loop2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: subs w8, w2, #1
-; CHECK-NEXT: b.lt .LBB1_6
-; CHECK-NEXT: // %bb.1: // %for.body.preheader
-; CHECK-NEXT: cmp w8, #2
-; CHECK-NEXT: b.ls .LBB1_3
-; CHECK-NEXT: // %bb.2: // %vector.memcheck
-; CHECK-NEXT: ubfiz x9, x8, #1, #32
-; CHECK-NEXT: add x9, x9, #2
-; CHECK-NEXT: add x10, x0, x9
-; CHECK-NEXT: add x9, x1, x9, lsl #2
-; CHECK-NEXT: cmp x10, x1
-; CHECK-NEXT: ccmp x9, x0, #0, hi
-; CHECK-NEXT: b.ls .LBB1_7
-; CHECK-NEXT: .LBB1_3:
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: mov x8, x1
-; CHECK-NEXT: mov x9, x0
-; CHECK-NEXT: .LBB1_4: // %for.body.preheader1
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
-; CHECK-NEXT: sub w10, w2, w10
-; CHECK-NEXT: fmov s1, w11
-; CHECK-NEXT: .LBB1_5: // %for.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldp s2, s3, [x8], #8
-; CHECK-NEXT: fcmp s2, s1
-; CHECK-NEXT: fcsel s4, s1, s2, gt
-; CHECK-NEXT: fcmp s2, #0.0
-; CHECK-NEXT: fcsel s2, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s1
-; CHECK-NEXT: fcsel s4, s1, s3, gt
-; CHECK-NEXT: fcmp s3, #0.0
-; CHECK-NEXT: fcvtzs s2, s2
-; CHECK-NEXT: fcsel s3, s0, s4, mi
-; CHECK-NEXT: subs w10, w10, #1
-; CHECK-NEXT: str b2, [x9]
-; CHECK-NEXT: fcvtzs s3, s3
-; CHECK-NEXT: stur b3, [x9, #1]
-; CHECK-NEXT: add x9, x9, #2
-; CHECK-NEXT: b.ne .LBB1_5
-; CHECK-NEXT: .LBB1_6: // %for.cond.cleanup
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB1_7: // %vector.ph
-; CHECK-NEXT: add x11, x8, #1
-; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-NEXT: and x10, x11, #0x1fffffffc
-; CHECK-NEXT: dup v0.4s, w8
-; CHECK-NEXT: and x12, x11, #0x1fffffffc
-; CHECK-NEXT: add x8, x1, x10, lsl #3
-; CHECK-NEXT: add x9, x0, x10, lsl #1
-; CHECK-NEXT: .LBB1_8: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x1], #32
-; CHECK-NEXT: subs x12, x12, #4
-; CHECK-NEXT: fcmgt v3.4s, v1.4s, v0.4s
-; CHECK-NEXT: fcmgt v4.4s, v2.4s, v0.4s
-; CHECK-NEXT: fcmlt v5.4s, v1.4s, #0.0
-; CHECK-NEXT: bsl v3.16b, v0.16b, v1.16b
-; CHECK-NEXT: bsl v4.16b, v0.16b, v2.16b
-; CHECK-NEXT: fcmlt v1.4s, v2.4s, #0.0
-; CHECK-NEXT: bic v2.16b, v3.16b, v5.16b
-; CHECK-NEXT: bic v1.16b, v4.16b, v1.16b
-; CHECK-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-NEXT: fcvtzs v1.4s, v1.4s
-; CHECK-NEXT: xtn v2.4h, v2.4s
-; CHECK-NEXT: xtn v1.4h, v1.4s
-; CHECK-NEXT: trn1 v1.8b, v2.8b, v1.8b
-; CHECK-NEXT: str d1, [x0], #8
-; CHECK-NEXT: b.ne .LBB1_8
-; CHECK-NEXT: // %bb.9: // %middle.block
-; CHECK-NEXT: cmp x11, x10
-; CHECK-NEXT: b.ne .LBB1_4
-; CHECK-NEXT: b .LBB1_6
+; CHECK-IAENABLED-LABEL: loop2:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: subs w8, w2, #1
+; CHECK-IAENABLED-NEXT: b.lt .LBB1_6
+; CHECK-IAENABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IAENABLED-NEXT: cmp w8, #2
+; CHECK-IAENABLED-NEXT: b.ls .LBB1_3
+; CHECK-IAENABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IAENABLED-NEXT: ubfiz x9, x8, #1, #32
+; CHECK-IAENABLED-NEXT: add x9, x9, #2
+; CHECK-IAENABLED-NEXT: add x10, x0, x9
+; CHECK-IAENABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IAENABLED-NEXT: cmp x10, x1
+; CHECK-IAENABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IAENABLED-NEXT: b.ls .LBB1_7
+; CHECK-IAENABLED-NEXT: .LBB1_3:
+; CHECK-IAENABLED-NEXT: mov w10, wzr
+; CHECK-IAENABLED-NEXT: mov x8, x1
+; CHECK-IAENABLED-NEXT: mov x9, x0
+; CHECK-IAENABLED-NEXT: .LBB1_4: // %for.body.preheader1
+; CHECK-IAENABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IAENABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: sub w10, w2, w10
+; CHECK-IAENABLED-NEXT: fmov s1, w11
+; CHECK-IAENABLED-NEXT: .LBB1_5: // %for.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ldp s2, s3, [x8], #8
+; CHECK-IAENABLED-NEXT: fcmp s2, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IAENABLED-NEXT: fcmp s2, #0.0
+; CHECK-IAENABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: fcvtzs s2, s2
+; CHECK-IAENABLED-NEXT: fcsel s3, s0, s4, mi
+; CHECK-IAENABLED-NEXT: subs w10, w10, #1
+; CHECK-IAENABLED-NEXT: str b2, [x9]
+; CHECK-IAENABLED-NEXT: fcvtzs s3, s3
+; CHECK-IAENABLED-NEXT: stur b3, [x9, #1]
+; CHECK-IAENABLED-NEXT: add x9, x9, #2
+; CHECK-IAENABLED-NEXT: b.ne .LBB1_5
+; CHECK-IAENABLED-NEXT: .LBB1_6: // %for.cond.cleanup
+; CHECK-IAENABLED-NEXT: ret
+; CHECK-IAENABLED-NEXT: .LBB1_7: // %vector.ph
+; CHECK-IAENABLED-NEXT: add x11, x8, #1
+; CHECK-IAENABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: dup v0.4s, w8
+; CHECK-IAENABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: add x8, x1, x10, lsl #3
+; CHECK-IAENABLED-NEXT: add x9, x0, x10, lsl #1
+; CHECK-IAENABLED-NEXT: .LBB1_8: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x1], #32
+; CHECK-IAENABLED-NEXT: subs x12, x12, #4
+; CHECK-IAENABLED-NEXT: fcmgt v3.4s, v1.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v4.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmlt v5.4s, v1.4s, #0.0
+; CHECK-IAENABLED-NEXT: bsl v3.16b, v0.16b, v1.16b
+; CHECK-IAENABLED-NEXT: bsl v4.16b, v0.16b, v2.16b
+; CHECK-IAENABLED-NEXT: fcmlt v1.4s, v2.4s, #0.0
+; CHECK-IAENABLED-NEXT: bic v2.16b, v3.16b, v5.16b
+; CHECK-IAENABLED-NEXT: bic v1.16b, v4.16b, v1.16b
+; CHECK-IAENABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: xtn v2.4h, v2.4s
+; CHECK-IAENABLED-NEXT: xtn v1.4h, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
+; CHECK-IAENABLED-NEXT: str d1, [x0], #8
+; CHECK-IAENABLED-NEXT: b.ne .LBB1_8
+; CHECK-IAENABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IAENABLED-NEXT: cmp x11, x10
+; CHECK-IAENABLED-NEXT: b.ne .LBB1_4
+; CHECK-IAENABLED-NEXT: b .LBB1_6
+;
+; CHECK-IADISABLED-LABEL: loop2:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: subs w8, w2, #1
+; CHECK-IADISABLED-NEXT: b.lt .LBB1_6
+; CHECK-IADISABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IADISABLED-NEXT: cmp w8, #2
+; CHECK-IADISABLED-NEXT: b.ls .LBB1_3
+; CHECK-IADISABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IADISABLED-NEXT: ubfiz x9, x8, #1, #32
+; CHECK-IADISABLED-NEXT: add x9, x9, #2
+; CHECK-IADISABLED-NEXT: add x10, x0, x9
+; CHECK-IADISABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IADISABLED-NEXT: cmp x10, x1
+; CHECK-IADISABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IADISABLED-NEXT: b.ls .LBB1_7
+; CHECK-IADISABLED-NEXT: .LBB1_3:
+; CHECK-IADISABLED-NEXT: mov w10, wzr
+; CHECK-IADISABLED-NEXT: mov x8, x1
+; CHECK-IADISABLED-NEXT: mov x9, x0
+; CHECK-IADISABLED-NEXT: .LBB1_4: // %for.body.preheader1
+; CHECK-IADISABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IADISABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: sub w10, w2, w10
+; CHECK-IADISABLED-NEXT: fmov s1, w11
+; CHECK-IADISABLED-NEXT: .LBB1_5: // %for.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp s2, s3, [x8], #8
+; CHECK-IADISABLED-NEXT: fcmp s2, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IADISABLED-NEXT: fcmp s2, #0.0
+; CHECK-IADISABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: fcvtzs s2, s2
+; CHECK-IADISABLED-NEXT: fcsel s3, s0, s4, mi
+; CHECK-IADISABLED-NEXT: subs w10, w10, #1
+; CHECK-IADISABLED-NEXT: str b2, [x9]
+; CHECK-IADISABLED-NEXT: fcvtzs s3, s3
+; CHECK-IADISABLED-NEXT: stur b3, [x9, #1]
+; CHECK-IADISABLED-NEXT: add x9, x9, #2
+; CHECK-IADISABLED-NEXT: b.ne .LBB1_5
+; CHECK-IADISABLED-NEXT: .LBB1_6: // %for.cond.cleanup
+; CHECK-IADISABLED-NEXT: ret
+; CHECK-IADISABLED-NEXT: .LBB1_7: // %vector.ph
+; CHECK-IADISABLED-NEXT: add x11, x8, #1
+; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: dup v0.4s, w8
+; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: add x8, x1, x10, lsl #3
+; CHECK-IADISABLED-NEXT: add x9, x0, x10, lsl #1
+; CHECK-IADISABLED-NEXT: .LBB1_8: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [x1], #32
+; CHECK-IADISABLED-NEXT: subs x12, x12, #4
+; CHECK-IADISABLED-NEXT: uzp1 v3.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v1.16b
+; CHECK-IADISABLED-NEXT: fcmlt v1.4s, v1.4s, #0.0
+; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v3.16b
+; CHECK-IADISABLED-NEXT: bic v1.16b, v4.16b, v1.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: xtn v2.4h, v2.4s
+; CHECK-IADISABLED-NEXT: xtn v1.4h, v1.4s
+; CHECK-IADISABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
+; CHECK-IADISABLED-NEXT: str d1, [x0], #8
+; CHECK-IADISABLED-NEXT: b.ne .LBB1_8
+; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IADISABLED-NEXT: cmp x11, x10
+; CHECK-IADISABLED-NEXT: b.ne .LBB1_4
+; CHECK-IADISABLED-NEXT: b .LBB1_6
entry:
%cmp19 = icmp sgt i32 %width, 0
br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
@@ -315,13 +394,938 @@ for.body: ; preds = %for.body.preheader3
}
define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
-; CHECK-LABEL: loop3:
+; CHECK-IAENABLED-LABEL: loop3:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: subs w8, w2, #1
+; CHECK-IAENABLED-NEXT: b.lt .LBB2_6
+; CHECK-IAENABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IAENABLED-NEXT: cmp w8, #2
+; CHECK-IAENABLED-NEXT: b.ls .LBB2_3
+; CHECK-IAENABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IAENABLED-NEXT: add x9, x8, w8, uxtw #1
+; CHECK-IAENABLED-NEXT: add x9, x9, #3
+; CHECK-IAENABLED-NEXT: add x10, x0, x9
+; CHECK-IAENABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IAENABLED-NEXT: cmp x10, x1
+; CHECK-IAENABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IAENABLED-NEXT: b.ls .LBB2_7
+; CHECK-IAENABLED-NEXT: .LBB2_3:
+; CHECK-IAENABLED-NEXT: mov w10, wzr
+; CHECK-IAENABLED-NEXT: mov x8, x1
+; CHECK-IAENABLED-NEXT: mov x9, x0
+; CHECK-IAENABLED-NEXT: .LBB2_4: // %for.body.preheader1
+; CHECK-IAENABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IAENABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: sub w10, w2, w10
+; CHECK-IAENABLED-NEXT: fmov s1, w11
+; CHECK-IAENABLED-NEXT: .LBB2_5: // %for.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ldp s2, s3, [x8]
+; CHECK-IAENABLED-NEXT: fcmp s2, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IAENABLED-NEXT: fcmp s2, #0.0
+; CHECK-IAENABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: ldr s3, [x8, #8]
+; CHECK-IAENABLED-NEXT: fcvtzs s2, s2
+; CHECK-IAENABLED-NEXT: add x8, x8, #12
+; CHECK-IAENABLED-NEXT: fcsel s4, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: str b2, [x9]
+; CHECK-IAENABLED-NEXT: fcsel s5, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: fcvtzs s4, s4
+; CHECK-IAENABLED-NEXT: fcsel s3, s0, s5, mi
+; CHECK-IAENABLED-NEXT: subs w10, w10, #1
+; CHECK-IAENABLED-NEXT: stur b4, [x9, #1]
+; CHECK-IAENABLED-NEXT: fcvtzs s3, s3
+; CHECK-IAENABLED-NEXT: stur b3, [x9, #2]
+; CHECK-IAENABLED-NEXT: add x9, x9, #3
+; CHECK-IAENABLED-NEXT: b.ne .LBB2_5
+; CHECK-IAENABLED-NEXT: .LBB2_6: // %for.cond.cleanup
+; CHECK-IAENABLED-NEXT: ret
+; CHECK-IAENABLED-NEXT: .LBB2_7: // %vector.ph
+; CHECK-IAENABLED-NEXT: add x11, x8, #1
+; CHECK-IAENABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: adrp x12, .LCPI2_0
+; CHECK-IAENABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: dup v0.4s, w8
+; CHECK-IAENABLED-NEXT: ldr q1, [x12, :lo12:.LCPI2_0]
+; CHECK-IAENABLED-NEXT: add x9, x10, x10, lsl #1
+; CHECK-IAENABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: add x8, x1, x9, lsl #2
+; CHECK-IAENABLED-NEXT: add x9, x0, x9
+; CHECK-IAENABLED-NEXT: .LBB2_8: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld3 { v2.4s, v3.4s, v4.4s }, [x1], #48
+; CHECK-IAENABLED-NEXT: subs x12, x12, #4
+; CHECK-IAENABLED-NEXT: fcmgt v5.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v6.4s, v3.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v7.4s, v4.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmlt v16.4s, v2.4s, #0.0
+; CHECK-IAENABLED-NEXT: fcmlt v17.4s, v3.4s, #0.0
+; CHECK-IAENABLED-NEXT: bsl v5.16b, v0.16b, v2.16b
+; CHECK-IAENABLED-NEXT: bsl v6.16b, v0.16b, v3.16b
+; CHECK-IAENABLED-NEXT: bsl v7.16b, v0.16b, v4.16b
+; CHECK-IAENABLED-NEXT: fcmlt v2.4s, v4.4s, #0.0
+; CHECK-IAENABLED-NEXT: bic v3.16b, v5.16b, v16.16b
+; CHECK-IAENABLED-NEXT: bic v4.16b, v6.16b, v17.16b
+; CHECK-IAENABLED-NEXT: bic v2.16b, v7.16b, v2.16b
+; CHECK-IAENABLED-NEXT: fcvtzs v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v4.4s, v4.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: xtn v5.4h, v3.4s
+; CHECK-IAENABLED-NEXT: xtn v6.4h, v4.4s
+; CHECK-IAENABLED-NEXT: xtn v7.4h, v2.4s
+; CHECK-IAENABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
+; CHECK-IAENABLED-NEXT: mov s3, v2.s[2]
+; CHECK-IAENABLED-NEXT: str d2, [x0]
+; CHECK-IAENABLED-NEXT: str s3, [x0, #8]
+; CHECK-IAENABLED-NEXT: add x0, x0, #12
+; CHECK-IAENABLED-NEXT: b.ne .LBB2_8
+; CHECK-IAENABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IAENABLED-NEXT: cmp x11, x10
+; CHECK-IAENABLED-NEXT: b.ne .LBB2_4
+; CHECK-IAENABLED-NEXT: b .LBB2_6
+;
+; CHECK-IADISABLED-LABEL: loop3:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: subs w8, w2, #1
+; CHECK-IADISABLED-NEXT: b.lt .LBB2_6
+; CHECK-IADISABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IADISABLED-NEXT: cmp w8, #2
+; CHECK-IADISABLED-NEXT: b.ls .LBB2_3
+; CHECK-IADISABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IADISABLED-NEXT: add x9, x8, w8, uxtw #1
+; CHECK-IADISABLED-NEXT: add x9, x9, #3
+; CHECK-IADISABLED-NEXT: add x10, x0, x9
+; CHECK-IADISABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IADISABLED-NEXT: cmp x10, x1
+; CHECK-IADISABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IADISABLED-NEXT: b.ls .LBB2_7
+; CHECK-IADISABLED-NEXT: .LBB2_3:
+; CHECK-IADISABLED-NEXT: mov w10, wzr
+; CHECK-IADISABLED-NEXT: mov x8, x1
+; CHECK-IADISABLED-NEXT: mov x9, x0
+; CHECK-IADISABLED-NEXT: .LBB2_4: // %for.body.preheader1
+; CHECK-IADISABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IADISABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: sub w10, w2, w10
+; CHECK-IADISABLED-NEXT: fmov s1, w11
+; CHECK-IADISABLED-NEXT: .LBB2_5: // %for.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp s2, s3, [x8]
+; CHECK-IADISABLED-NEXT: fcmp s2, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IADISABLED-NEXT: fcmp s2, #0.0
+; CHECK-IADISABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: ldr s3, [x8, #8]
+; CHECK-IADISABLED-NEXT: fcvtzs s2, s2
+; CHECK-IADISABLED-NEXT: add x8, x8, #12
+; CHECK-IADISABLED-NEXT: fcsel s4, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: str b2, [x9]
+; CHECK-IADISABLED-NEXT: fcsel s5, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: fcvtzs s4, s4
+; CHECK-IADISABLED-NEXT: fcsel s3, s0, s5, mi
+; CHECK-IADISABLED-NEXT: subs w10, w10, #1
+; CHECK-IADISABLED-NEXT: stur b4, [x9, #1]
+; CHECK-IADISABLED-NEXT: fcvtzs s3, s3
+; CHECK-IADISABLED-NEXT: stur b3, [x9, #2]
+; CHECK-IADISABLED-NEXT: add x9, x9, #3
+; CHECK-IADISABLED-NEXT: b.ne .LBB2_5
+; CHECK-IADISABLED-NEXT: .LBB2_6: // %for.cond.cleanup
+; CHECK-IADISABLED-NEXT: ret
+; CHECK-IADISABLED-NEXT: .LBB2_7: // %vector.ph
+; CHECK-IADISABLED-NEXT: add x11, x8, #1
+; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: adrp x12, .LCPI2_0
+; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: dup v0.4s, w8
+; CHECK-IADISABLED-NEXT: ldr q1, [x12, :lo12:.LCPI2_0]
+; CHECK-IADISABLED-NEXT: add x9, x10, x10, lsl #1
+; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: add x8, x1, x9, lsl #2
+; CHECK-IADISABLED-NEXT: add x9, x0, x9
+; CHECK-IADISABLED-NEXT: .LBB2_8: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q2, q4, [x1]
+; CHECK-IADISABLED-NEXT: subs x12, x12, #4
+; CHECK-IADISABLED-NEXT: ldr q6, [x1, #32]
+; CHECK-IADISABLED-NEXT: add x1, x1, #48
+; CHECK-IADISABLED-NEXT: mov v3.16b, v2.16b
+; CHECK-IADISABLED-NEXT: rev64 v5.4s, v4.4s
+; CHECK-IADISABLED-NEXT: mov v3.s[1], v2.s[3]
+; CHECK-IADISABLED-NEXT: mov v5.s[0], v2.s[1]
+; CHECK-IADISABLED-NEXT: mov v3.s[2], v4.s[2]
+; CHECK-IADISABLED-NEXT: mov v4.s[0], v2.s[2]
+; CHECK-IADISABLED-NEXT: mov v5.s[3], v6.s[2]
+; CHECK-IADISABLED-NEXT: mov v4.s[2], v6.s[0]
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v6.s[1]
+; CHECK-IADISABLED-NEXT: mov v4.s[3], v6.s[3]
+; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v6.16b, v0.16b, v5.16b
+; CHECK-IADISABLED-NEXT: fcmlt v5.4s, v5.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v7.16b, v0.16b, v4.16b
+; CHECK-IADISABLED-NEXT: fcmlt v4.4s, v4.4s, #0.0
+; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v3.16b
+; CHECK-IADISABLED-NEXT: bic v3.16b, v6.16b, v5.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: bic v4.16b, v7.16b, v4.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: xtn v5.4h, v2.4s
+; CHECK-IADISABLED-NEXT: xtn v6.4h, v3.4s
+; CHECK-IADISABLED-NEXT: xtn v7.4h, v4.4s
+; CHECK-IADISABLED-NEXT: tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
+; CHECK-IADISABLED-NEXT: mov s3, v2.s[2]
+; CHECK-IADISABLED-NEXT: str d2, [x0]
+; CHECK-IADISABLED-NEXT: str s3, [x0, #8]
+; CHECK-IADISABLED-NEXT: add x0, x0, #12
+; CHECK-IADISABLED-NEXT: b.ne .LBB2_8
+; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IADISABLED-NEXT: cmp x11, x10
+; CHECK-IADISABLED-NEXT: b.ne .LBB2_4
+; CHECK-IADISABLED-NEXT: b .LBB2_6
+entry:
+ %cmp29 = icmp sgt i32 %width, 0
+ br i1 %cmp29, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %0 = add i32 %width, -1
+ %1 = zext i32 %0 to i64
+ %2 = add nuw nsw i64 %1, 1
+ %min.iters.check = icmp ult i32 %0, 3
+ br i1 %min.iters.check, label %for.body.preheader46, label %vector.memcheck
+
+vector.memcheck: ; preds = %for.body.preheader
+ %3 = add i32 %width, -1
+ %4 = zext i32 %3 to i64
+ %5 = mul nuw nsw i64 %4, 3
+ %6 = add nuw nsw i64 %5, 3
+ %scevgep = getelementptr i8, ptr %dst, i64 %6
+ %scevgep34 = getelementptr float, ptr %data, i64 %6
+ %bound0 = icmp ugt ptr %scevgep34, %dst
+ %bound1 = icmp ugt ptr %scevgep, %data
+ %found.conflict = and i1 %bound0, %bound1
+ br i1 %found.conflict, label %for.body.preheader46, label %vector.ph
+
+vector.ph: ; preds = %vector.memcheck
+ %n.vec = and i64 %2, 8589934588
+ %ind.end = trunc i64 %n.vec to i32
+ %7 = mul nuw nsw i64 %n.vec, 3
+ %ind.end37 = getelementptr float, ptr %data, i64 %7
+ %8 = mul nuw nsw i64 %n.vec, 3
+ %ind.end39 = getelementptr i8, ptr %dst, i64 %8
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %9 = mul i64 %index, 3
+ %next.gep = getelementptr float, ptr %data, i64 %9
+ %10 = mul i64 %index, 3
+ %wide.vec = load <12 x float>, ptr %next.gep, align 4
+ %strided.vec = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %strided.vec44 = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %strided.vec45 = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %11 = fcmp olt <4 x float> %strided.vec, zeroinitializer
+ %12 = fcmp ogt <4 x float> %strided.vec, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %13 = select <4 x i1> %12, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec
+ %14 = select <4 x i1> %11, <4 x float> zeroinitializer, <4 x float> %13
+ %15 = fptoui <4 x float> %14 to <4 x i8>
+ %16 = fcmp olt <4 x float> %strided.vec44, zeroinitializer
+ %17 = fcmp ogt <4 x float> %strided.vec44, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %18 = select <4 x i1> %17, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec44
+ %19 = select <4 x i1> %16, <4 x float> zeroinitializer, <4 x float> %18
+ %20 = fptoui <4 x float> %19 to <4 x i8>
+ %21 = fcmp olt <4 x float> %strided.vec45, zeroinitializer
+ %22 = fcmp ogt <4 x float> %strided.vec45, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %23 = select <4 x i1> %22, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec45
+ %24 = select <4 x i1> %21, <4 x float> zeroinitializer, <4 x float> %23
+ %25 = fptoui <4 x float> %24 to <4 x i8>
+ %26 = getelementptr inbounds i8, ptr %dst, i64 %10
+ %27 = shufflevector <4 x i8> %15, <4 x i8> %20, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %28 = shufflevector <4 x i8> %25, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %interleaved.vec = shufflevector <8 x i8> %27, <8 x i8> %28, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ store <12 x i8> %interleaved.vec, ptr %26, align 1
+ %index.next = add nuw i64 %index, 4
+ %29 = icmp eq i64 %index.next, %n.vec
+ br i1 %29, label %middle.block, label %vector.body
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %2, %n.vec
+ br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader46
+
+for.body.preheader46: ; preds = %vector.memcheck, %for.body.preheader, %middle.block
+ %i.032.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %ind.end, %middle.block ]
+ %src.031.ph = phi ptr [ %data, %vector.memcheck ], [ %data, %for.body.preheader ], [ %ind.end37, %middle.block ]
+ %dst.addr.030.ph = phi ptr [ %dst, %vector.memcheck ], [ %dst, %for.body.preheader ], [ %ind.end39, %middle.block ]
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader46, %for.body
+ %i.032 = phi i32 [ %inc, %for.body ], [ %i.032.ph, %for.body.preheader46 ]
+ %src.031 = phi ptr [ %add.ptr, %for.body ], [ %src.031.ph, %for.body.preheader46 ]
+ %dst.addr.030 = phi ptr [ %add.ptr10, %for.body ], [ %dst.addr.030.ph, %for.body.preheader46 ]
+ %30 = load float, ptr %src.031, align 4
+ %cmp.i = fcmp olt float %30, 0.000000e+00
+ %cmp1.i = fcmp ogt float %30, 2.550000e+02
+ %.x.i = select i1 %cmp1.i, float 2.550000e+02, float %30
+ %retval.0.i = select i1 %cmp.i, float 0.000000e+00, float %.x.i
+ %conv = fptoui float %retval.0.i to i8
+ store i8 %conv, ptr %dst.addr.030, align 1
+ %arrayidx2 = getelementptr inbounds float, ptr %src.031, i64 1
+ %31 = load float, ptr %arrayidx2, align 4
+ %cmp.i21 = fcmp olt float %31, 0.000000e+00
+ %cmp1.i22 = fcmp ogt float %31, 2.550000e+02
+ %.x.i23 = select i1 %cmp1.i22, float 2.550000e+02, float %31
+ %retval.0.i24 = select i1 %cmp.i21, float 0.000000e+00, float %.x.i23
+ %conv4 = fptoui float %retval.0.i24 to i8
+ %arrayidx5 = getelementptr inbounds i8, ptr %dst.addr.030, i64 1
+ store i8 %conv4, ptr %arrayidx5, align 1
+ %arrayidx6 = getelementptr inbounds float, ptr %src.031, i64 2
+ %32 = load float, ptr %arrayidx6, align 4
+ %cmp.i25 = fcmp olt float %32, 0.000000e+00
+ %cmp1.i26 = fcmp ogt float %32, 2.550000e+02
+ %.x.i27 = select i1 %cmp1.i26, float 2.550000e+02, float %32
+ %retval.0.i28 = select i1 %cmp.i25, float 0.000000e+00, float %.x.i27
+ %conv8 = fptoui float %retval.0.i28 to i8
+ %arrayidx9 = getelementptr inbounds i8, ptr %dst.addr.030, i64 2
+ store i8 %conv8, ptr %arrayidx9, align 1
+ %add.ptr = getelementptr inbounds float, ptr %src.031, i64 3
+ %add.ptr10 = getelementptr inbounds i8, ptr %dst.addr.030, i64 3
+ %inc = add nuw nsw i32 %i.032, 1
+ %exitcond.not = icmp eq i32 %inc, %width
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
+; CHECK-IAENABLED-LABEL: loop4:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: subs w8, w2, #1
+; CHECK-IAENABLED-NEXT: b.lt .LBB3_6
+; CHECK-IAENABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IAENABLED-NEXT: cmp w8, #2
+; CHECK-IAENABLED-NEXT: b.ls .LBB3_3
+; CHECK-IAENABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IAENABLED-NEXT: ubfiz x9, x8, #2, #32
+; CHECK-IAENABLED-NEXT: add x9, x9, #4
+; CHECK-IAENABLED-NEXT: add x10, x0, x9
+; CHECK-IAENABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IAENABLED-NEXT: cmp x10, x1
+; CHECK-IAENABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IAENABLED-NEXT: b.ls .LBB3_7
+; CHECK-IAENABLED-NEXT: .LBB3_3:
+; CHECK-IAENABLED-NEXT: mov w10, wzr
+; CHECK-IAENABLED-NEXT: mov x8, x1
+; CHECK-IAENABLED-NEXT: mov x9, x0
+; CHECK-IAENABLED-NEXT: .LBB3_4: // %for.body.preheader1
+; CHECK-IAENABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IAENABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: sub w10, w2, w10
+; CHECK-IAENABLED-NEXT: fmov s1, w11
+; CHECK-IAENABLED-NEXT: .LBB3_5: // %for.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ldp s2, s3, [x8]
+; CHECK-IAENABLED-NEXT: fcmp s2, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IAENABLED-NEXT: fcmp s2, #0.0
+; CHECK-IAENABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: fcvtzs s2, s2
+; CHECK-IAENABLED-NEXT: ldp s3, s5, [x8, #8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: fcsel s4, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: str b2, [x9]
+; CHECK-IAENABLED-NEXT: fcvtzs s4, s4
+; CHECK-IAENABLED-NEXT: fcsel s6, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: fcsel s3, s0, s6, mi
+; CHECK-IAENABLED-NEXT: fcmp s5, s1
+; CHECK-IAENABLED-NEXT: stur b4, [x9, #1]
+; CHECK-IAENABLED-NEXT: fcsel s6, s1, s5, gt
+; CHECK-IAENABLED-NEXT: fcmp s5, #0.0
+; CHECK-IAENABLED-NEXT: fcvtzs s3, s3
+; CHECK-IAENABLED-NEXT: fcsel s5, s0, s6, mi
+; CHECK-IAENABLED-NEXT: subs w10, w10, #1
+; CHECK-IAENABLED-NEXT: stur b3, [x9, #2]
+; CHECK-IAENABLED-NEXT: fcvtzs s5, s5
+; CHECK-IAENABLED-NEXT: stur b5, [x9, #3]
+; CHECK-IAENABLED-NEXT: add x9, x9, #4
+; CHECK-IAENABLED-NEXT: b.ne .LBB3_5
+; CHECK-IAENABLED-NEXT: .LBB3_6: // %for.cond.cleanup
+; CHECK-IAENABLED-NEXT: ret
+; CHECK-IAENABLED-NEXT: .LBB3_7: // %vector.ph
+; CHECK-IAENABLED-NEXT: add x11, x8, #1
+; CHECK-IAENABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: adrp x12, .LCPI3_0
+; CHECK-IAENABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: dup v0.4s, w8
+; CHECK-IAENABLED-NEXT: ldr q1, [x12, :lo12:.LCPI3_0]
+; CHECK-IAENABLED-NEXT: add x8, x1, x10, lsl #4
+; CHECK-IAENABLED-NEXT: add x9, x0, x10, lsl #2
+; CHECK-IAENABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: .LBB3_8: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld4 { v2.4s, v3.4s, v4.4s, v5.4s }, [x1], #64
+; CHECK-IAENABLED-NEXT: subs x12, x12, #4
+; CHECK-IAENABLED-NEXT: fcmgt v6.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v7.4s, v3.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v16.4s, v4.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v17.4s, v5.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmlt v18.4s, v2.4s, #0.0
+; CHECK-IAENABLED-NEXT: fcmlt v19.4s, v3.4s, #0.0
+; CHECK-IAENABLED-NEXT: fcmlt v20.4s, v4.4s, #0.0
+; CHECK-IAENABLED-NEXT: bsl v6.16b, v0.16b, v2.16b
+; CHECK-IAENABLED-NEXT: bsl v7.16b, v0.16b, v3.16b
+; CHECK-IAENABLED-NEXT: bsl v16.16b, v0.16b, v4.16b
+; CHECK-IAENABLED-NEXT: bsl v17.16b, v0.16b, v5.16b
+; CHECK-IAENABLED-NEXT: fcmlt v2.4s, v5.4s, #0.0
+; CHECK-IAENABLED-NEXT: bic v3.16b, v6.16b, v18.16b
+; CHECK-IAENABLED-NEXT: bic v4.16b, v7.16b, v19.16b
+; CHECK-IAENABLED-NEXT: bic v5.16b, v16.16b, v20.16b
+; CHECK-IAENABLED-NEXT: bic v2.16b, v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: fcvtzs v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v4.4s, v4.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v5.4s, v5.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: xtn v16.4h, v3.4s
+; CHECK-IAENABLED-NEXT: xtn v17.4h, v4.4s
+; CHECK-IAENABLED-NEXT: xtn v18.4h, v5.4s
+; CHECK-IAENABLED-NEXT: xtn v19.4h, v2.4s
+; CHECK-IAENABLED-NEXT: tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-IAENABLED-NEXT: str q2, [x0], #16
+; CHECK-IAENABLED-NEXT: b.ne .LBB3_8
+; CHECK-IAENABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IAENABLED-NEXT: cmp x11, x10
+; CHECK-IAENABLED-NEXT: b.ne .LBB3_4
+; CHECK-IAENABLED-NEXT: b .LBB3_6
+;
+; CHECK-IADISABLED-LABEL: loop4:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: subs w8, w2, #1
+; CHECK-IADISABLED-NEXT: b.lt .LBB3_6
+; CHECK-IADISABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IADISABLED-NEXT: cmp w8, #2
+; CHECK-IADISABLED-NEXT: b.ls .LBB3_3
+; CHECK-IADISABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IADISABLED-NEXT: ubfiz x9, x8, #2, #32
+; CHECK-IADISABLED-NEXT: add x9, x9, #4
+; CHECK-IADISABLED-NEXT: add x10, x0, x9
+; CHECK-IADISABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IADISABLED-NEXT: cmp x10, x1
+; CHECK-IADISABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IADISABLED-NEXT: b.ls .LBB3_7
+; CHECK-IADISABLED-NEXT: .LBB3_3:
+; CHECK-IADISABLED-NEXT: mov w10, wzr
+; CHECK-IADISABLED-NEXT: mov x8, x1
+; CHECK-IADISABLED-NEXT: mov x9, x0
+; CHECK-IADISABLED-NEXT: .LBB3_4: // %for.body.preheader1
+; CHECK-IADISABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IADISABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: sub w10, w2, w10
+; CHECK-IADISABLED-NEXT: fmov s1, w11
+; CHECK-IADISABLED-NEXT: .LBB3_5: // %for.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp s2, s3, [x8]
+; CHECK-IADISABLED-NEXT: fcmp s2, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IADISABLED-NEXT: fcmp s2, #0.0
+; CHECK-IADISABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: fcvtzs s2, s2
+; CHECK-IADISABLED-NEXT: ldp s3, s5, [x8, #8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: fcsel s4, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: str b2, [x9]
+; CHECK-IADISABLED-NEXT: fcvtzs s4, s4
+; CHECK-IADISABLED-NEXT: fcsel s6, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: fcsel s3, s0, s6, mi
+; CHECK-IADISABLED-NEXT: fcmp s5, s1
+; CHECK-IADISABLED-NEXT: stur b4, [x9, #1]
+; CHECK-IADISABLED-NEXT: fcsel s6, s1, s5, gt
+; CHECK-IADISABLED-NEXT: fcmp s5, #0.0
+; CHECK-IADISABLED-NEXT: fcvtzs s3, s3
+; CHECK-IADISABLED-NEXT: fcsel s5, s0, s6, mi
+; CHECK-IADISABLED-NEXT: subs w10, w10, #1
+; CHECK-IADISABLED-NEXT: stur b3, [x9, #2]
+; CHECK-IADISABLED-NEXT: fcvtzs s5, s5
+; CHECK-IADISABLED-NEXT: stur b5, [x9, #3]
+; CHECK-IADISABLED-NEXT: add x9, x9, #4
+; CHECK-IADISABLED-NEXT: b.ne .LBB3_5
+; CHECK-IADISABLED-NEXT: .LBB3_6: // %for.cond.cleanup
+; CHECK-IADISABLED-NEXT: ret
+; CHECK-IADISABLED-NEXT: .LBB3_7: // %vector.ph
+; CHECK-IADISABLED-NEXT: add x11, x8, #1
+; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: adrp x12, .LCPI3_0
+; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: dup v0.4s, w8
+; CHECK-IADISABLED-NEXT: ldr q1, [x12, :lo12:.LCPI3_0]
+; CHECK-IADISABLED-NEXT: add x8, x1, x10, lsl #4
+; CHECK-IADISABLED-NEXT: add x9, x0, x10, lsl #2
+; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: .LBB3_8: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1, #32]
+; CHECK-IADISABLED-NEXT: subs x12, x12, #4
+; CHECK-IADISABLED-NEXT: ldp q6, q5, [x1], #64
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: trn1 v19.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v7.4s, v6.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip1 v16.4s, v6.4s, v5.4s
+; CHECK-IADISABLED-NEXT: trn2 v18.4s, v6.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip2 v5.4s, v6.4s, v5.4s
+; CHECK-IADISABLED-NEXT: ext v17.16b, v3.16b, v4.16b, #8
+; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v7.4s, v6.4s
+; CHECK-IADISABLED-NEXT: mov v18.d[1], v4.d[1]
+; CHECK-IADISABLED-NEXT: mov v5.d[1], v19.d[1]
+; CHECK-IADISABLED-NEXT: mov v16.d[1], v17.d[1]
+; CHECK-IADISABLED-NEXT: mov v3.d[1], v2.d[1]
+; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v18.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmlt v17.4s, v18.4s, #0.0
+; CHECK-IADISABLED-NEXT: fcmgt v6.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v16.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v7.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v18.16b
+; CHECK-IADISABLED-NEXT: bsl v6.16b, v0.16b, v5.16b
+; CHECK-IADISABLED-NEXT: fcmlt v5.4s, v5.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v16.16b
+; CHECK-IADISABLED-NEXT: fcmlt v16.4s, v16.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v7.16b, v0.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
+; CHECK-IADISABLED-NEXT: bic v4.16b, v4.16b, v17.16b
+; CHECK-IADISABLED-NEXT: bic v5.16b, v6.16b, v5.16b
+; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v16.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: bic v3.16b, v7.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v5.4s, v5.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: xtn v16.4h, v2.4s
+; CHECK-IADISABLED-NEXT: xtn v17.4h, v4.4s
+; CHECK-IADISABLED-NEXT: xtn v18.4h, v5.4s
+; CHECK-IADISABLED-NEXT: xtn v19.4h, v3.4s
+; CHECK-IADISABLED-NEXT: tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-IADISABLED-NEXT: str q2, [x0], #16
+; CHECK-IADISABLED-NEXT: b.ne .LBB3_8
+; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IADISABLED-NEXT: cmp x11, x10
+; CHECK-IADISABLED-NEXT: b.ne .LBB3_4
+; CHECK-IADISABLED-NEXT: b .LBB3_6
+entry:
+ %cmp39 = icmp sgt i32 %width, 0
+ br i1 %cmp39, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %0 = add i32 %width, -1
+ %1 = zext i32 %0 to i64
+ %2 = add nuw nsw i64 %1, 1
+ %min.iters.check = icmp ult i32 %0, 3
+ br i1 %min.iters.check, label %for.body.preheader57, label %vector.memcheck
+
+vector.memcheck: ; preds = %for.body.preheader
+ %3 = add i32 %width, -1
+ %4 = zext i32 %3 to i64
+ %5 = shl nuw nsw i64 %4, 2
+ %6 = add nuw nsw i64 %5, 4
+ %scevgep = getelementptr i8, ptr %dst, i64 %6
+ %scevgep44 = getelementptr float, ptr %data, i64 %6
+ %bound0 = icmp ugt ptr %scevgep44, %dst
+ %bound1 = icmp ugt ptr %scevgep, %data
+ %found.conflict = and i1 %bound0, %bound1
+ br i1 %found.conflict, label %for.body.preheader57, label %vector.ph
+
+vector.ph: ; preds = %vector.memcheck
+ %n.vec = and i64 %2, 8589934588
+ %ind.end = trunc i64 %n.vec to i32
+ %7 = shl nuw nsw i64 %n.vec, 2
+ %ind.end47 = getelementptr float, ptr %data, i64 %7
+ %8 = shl nuw nsw i64 %n.vec, 2
+ %ind.end49 = getelementptr i8, ptr %dst, i64 %8
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %9 = shl i64 %index, 2
+ %next.gep = getelementptr float, ptr %data, i64 %9
+ %10 = shl i64 %index, 2
+ %wide.vec = load <16 x float>, ptr %next.gep, align 4
+ %strided.vec = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %strided.vec54 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %strided.vec55 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %strided.vec56 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %11 = fcmp olt <4 x float> %strided.vec, zeroinitializer
+ %12 = fcmp ogt <4 x float> %strided.vec, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %13 = select <4 x i1> %12, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec
+ %14 = select <4 x i1> %11, <4 x float> zeroinitializer, <4 x float> %13
+ %15 = fptoui <4 x float> %14 to <4 x i8>
+ %16 = fcmp olt <4 x float> %strided.vec54, zeroinitializer
+ %17 = fcmp ogt <4 x float> %strided.vec54, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %18 = select <4 x i1> %17, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec54
+ %19 = select <4 x i1> %16, <4 x float> zeroinitializer, <4 x float> %18
+ %20 = fptoui <4 x float> %19 to <4 x i8>
+ %21 = fcmp olt <4 x float> %strided.vec55, zeroinitializer
+ %22 = fcmp ogt <4 x float> %strided.vec55, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %23 = select <4 x i1> %22, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec55
+ %24 = select <4 x i1> %21, <4 x float> zeroinitializer, <4 x float> %23
+ %25 = fptoui <4 x float> %24 to <4 x i8>
+ %26 = fcmp olt <4 x float> %strided.vec56, zeroinitializer
+ %27 = fcmp ogt <4 x float> %strided.vec56, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %28 = select <4 x i1> %27, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec56
+ %29 = select <4 x i1> %26, <4 x float> zeroinitializer, <4 x float> %28
+ %30 = fptoui <4 x float> %29 to <4 x i8>
+ %31 = getelementptr inbounds i8, ptr %dst, i64 %10
+ %32 = shufflevector <4 x i8> %15, <4 x i8> %20, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %33 = shufflevector <4 x i8> %25, <4 x i8> %30, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %interleaved.vec = shufflevector <8 x i8> %32, <8 x i8> %33, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ store <16 x i8> %interleaved.vec, ptr %31, align 1
+ %index.next = add nuw i64 %index, 4
+ %34 = icmp eq i64 %index.next, %n.vec
+ br i1 %34, label %middle.block, label %vector.body
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %2, %n.vec
+ br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader57
+
+for.body.preheader57: ; preds = %vector.memcheck, %for.body.preheader, %middle.block
+ %i.042.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %ind.end, %middle.block ]
+ %src.041.ph = phi ptr [ %data, %vector.memcheck ], [ %data, %for.body.preheader ], [ %ind.end47, %middle.block ]
+ %dst.addr.040.ph = phi ptr [ %dst, %vector.memcheck ], [ %dst, %for.body.preheader ], [ %ind.end49, %middle.block ]
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader57, %for.body
+ %i.042 = phi i32 [ %inc, %for.body ], [ %i.042.ph, %for.body.preheader57 ]
+ %src.041 = phi ptr [ %add.ptr, %for.body ], [ %src.041.ph, %for.body.preheader57 ]
+ %dst.addr.040 = phi ptr [ %add.ptr14, %for.body ], [ %dst.addr.040.ph, %for.body.preheader57 ]
+ %35 = load float, ptr %src.041, align 4
+ %cmp.i = fcmp olt float %35, 0.000000e+00
+ %cmp1.i = fcmp ogt float %35, 2.550000e+02
+ %.x.i = select i1 %cmp1.i, float 2.550000e+02, float %35
+ %retval.0.i = select i1 %cmp.i, float 0.000000e+00, float %.x.i
+ %conv = fptoui float %retval.0.i to i8
+ store i8 %conv, ptr %dst.addr.040, align 1
+ %arrayidx2 = getelementptr inbounds float, ptr %src.041, i64 1
+ %36 = load float, ptr %arrayidx2, align 4
+ %cmp.i27 = fcmp olt float %36, 0.000000e+00
+ %cmp1.i28 = fcmp ogt float %36, 2.550000e+02
+ %.x.i29 = select i1 %cmp1.i28, float 2.550000e+02, float %36
+ %retval.0.i30 = select i1 %cmp.i27, float 0.000000e+00, float %.x.i29
+ %conv4 = fptoui float %retval.0.i30 to i8
+ %arrayidx5 = getelementptr inbounds i8, ptr %dst.addr.040, i64 1
+ store i8 %conv4, ptr %arrayidx5, align 1
+ %arrayidx6 = getelementptr inbounds float, ptr %src.041, i64 2
+ %37 = load float, ptr %arrayidx6, align 4
+ %cmp.i31 = fcmp olt float %37, 0.000000e+00
+ %cmp1.i32 = fcmp ogt float %37, 2.550000e+02
+ %.x.i33 = select i1 %cmp1.i32, float 2.550000e+02, float %37
+ %retval.0.i34 = select i1 %cmp.i31, float 0.000000e+00, float %.x.i33
+ %conv8 = fptoui float %retval.0.i34 to i8
+ %arrayidx9 = getelementptr inbounds i8, ptr %dst.addr.040, i64 2
+ store i8 %conv8, ptr %arrayidx9, align 1
+ %arrayidx10 = getelementptr inbounds float, ptr %src.041, i64 3
+ %38 = load float, ptr %arrayidx10, align 4
+ %cmp.i35 = fcmp olt float %38, 0.000000e+00
+ %cmp1.i36 = fcmp ogt float %38, 2.550000e+02
+ %.x.i37 = select i1 %cmp1.i36, float 2.550000e+02, float %38
+ %retval.0.i38 = select i1 %cmp.i35, float 0.000000e+00, float %.x.i37
+ %conv12 = fptoui float %retval.0.i38 to i8
+ %arrayidx13 = getelementptr inbounds i8, ptr %dst.addr.040, i64 3
+ store i8 %conv12, ptr %arrayidx13, align 1
+ %add.ptr = getelementptr inbounds float, ptr %src.041, i64 4
+ %add.ptr14 = getelementptr inbounds i8, ptr %dst.addr.040, i64 4
+ %inc = add nuw nsw i32 %i.042, 1
+ %exitcond.not = icmp eq i32 %inc, %width
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @loop2_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
+; CHECK-IAENABLED-LABEL: loop2_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: subs w8, w2, #1
+; CHECK-IAENABLED-NEXT: b.lt .LBB4_6
+; CHECK-IAENABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IAENABLED-NEXT: cmp w8, #2
+; CHECK-IAENABLED-NEXT: b.ls .LBB4_3
+; CHECK-IAENABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IAENABLED-NEXT: ubfiz x9, x8, #1, #32
+; CHECK-IAENABLED-NEXT: add x9, x9, #2
+; CHECK-IAENABLED-NEXT: add x10, x0, x9
+; CHECK-IAENABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IAENABLED-NEXT: cmp x10, x1
+; CHECK-IAENABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IAENABLED-NEXT: b.ls .LBB4_7
+; CHECK-IAENABLED-NEXT: .LBB4_3:
+; CHECK-IAENABLED-NEXT: mov w10, wzr
+; CHECK-IAENABLED-NEXT: mov x8, x1
+; CHECK-IAENABLED-NEXT: mov x9, x0
+; CHECK-IAENABLED-NEXT: .LBB4_4: // %for.body.preheader1
+; CHECK-IAENABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IAENABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: sub w10, w2, w10
+; CHECK-IAENABLED-NEXT: fmov s1, w11
+; CHECK-IAENABLED-NEXT: .LBB4_5: // %for.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ldp s2, s3, [x8], #8
+; CHECK-IAENABLED-NEXT: fcmp s2, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IAENABLED-NEXT: fcmp s2, #0.0
+; CHECK-IAENABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IAENABLED-NEXT: fcmp s3, s1
+; CHECK-IAENABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IAENABLED-NEXT: fcmp s3, #0.0
+; CHECK-IAENABLED-NEXT: fcvtzs s2, s2
+; CHECK-IAENABLED-NEXT: fcsel s3, s0, s4, mi
+; CHECK-IAENABLED-NEXT: subs w10, w10, #1
+; CHECK-IAENABLED-NEXT: str b2, [x9]
+; CHECK-IAENABLED-NEXT: fcvtzs s3, s3
+; CHECK-IAENABLED-NEXT: stur b3, [x9, #1]
+; CHECK-IAENABLED-NEXT: add x9, x9, #2
+; CHECK-IAENABLED-NEXT: b.ne .LBB4_5
+; CHECK-IAENABLED-NEXT: .LBB4_6: // %for.cond.cleanup
+; CHECK-IAENABLED-NEXT: ret
+; CHECK-IAENABLED-NEXT: .LBB4_7: // %vector.ph
+; CHECK-IAENABLED-NEXT: add x11, x8, #1
+; CHECK-IAENABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IAENABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: dup v0.4s, w8
+; CHECK-IAENABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IAENABLED-NEXT: add x8, x1, x10, lsl #3
+; CHECK-IAENABLED-NEXT: add x9, x0, x10, lsl #1
+; CHECK-IAENABLED-NEXT: .LBB4_8: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x1], #32
+; CHECK-IAENABLED-NEXT: subs x12, x12, #4
+; CHECK-IAENABLED-NEXT: fcmgt v3.4s, v1.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmgt v4.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fcmlt v5.4s, v1.4s, #0.0
+; CHECK-IAENABLED-NEXT: bsl v3.16b, v0.16b, v1.16b
+; CHECK-IAENABLED-NEXT: bsl v4.16b, v0.16b, v2.16b
+; CHECK-IAENABLED-NEXT: fcmlt v1.4s, v2.4s, #0.0
+; CHECK-IAENABLED-NEXT: bic v2.16b, v3.16b, v5.16b
+; CHECK-IAENABLED-NEXT: bic v1.16b, v4.16b, v1.16b
+; CHECK-IAENABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: xtn v2.4h, v2.4s
+; CHECK-IAENABLED-NEXT: xtn v1.4h, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
+; CHECK-IAENABLED-NEXT: str d1, [x0], #8
+; CHECK-IAENABLED-NEXT: b.ne .LBB4_8
+; CHECK-IAENABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IAENABLED-NEXT: cmp x11, x10
+; CHECK-IAENABLED-NEXT: b.ne .LBB4_4
+; CHECK-IAENABLED-NEXT: b .LBB4_6
+;
+; CHECK-IADISABLED-LABEL: loop2_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: subs w8, w2, #1
+; CHECK-IADISABLED-NEXT: b.lt .LBB4_6
+; CHECK-IADISABLED-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-IADISABLED-NEXT: cmp w8, #2
+; CHECK-IADISABLED-NEXT: b.ls .LBB4_3
+; CHECK-IADISABLED-NEXT: // %bb.2: // %vector.memcheck
+; CHECK-IADISABLED-NEXT: ubfiz x9, x8, #1, #32
+; CHECK-IADISABLED-NEXT: add x9, x9, #2
+; CHECK-IADISABLED-NEXT: add x10, x0, x9
+; CHECK-IADISABLED-NEXT: add x9, x1, x9, lsl #2
+; CHECK-IADISABLED-NEXT: cmp x10, x1
+; CHECK-IADISABLED-NEXT: ccmp x9, x0, #0, hi
+; CHECK-IADISABLED-NEXT: b.ls .LBB4_7
+; CHECK-IADISABLED-NEXT: .LBB4_3:
+; CHECK-IADISABLED-NEXT: mov w10, wzr
+; CHECK-IADISABLED-NEXT: mov x8, x1
+; CHECK-IADISABLED-NEXT: mov x9, x0
+; CHECK-IADISABLED-NEXT: .LBB4_4: // %for.body.preheader1
+; CHECK-IADISABLED-NEXT: movi d0, #0000000000000000
+; CHECK-IADISABLED-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: sub w10, w2, w10
+; CHECK-IADISABLED-NEXT: fmov s1, w11
+; CHECK-IADISABLED-NEXT: .LBB4_5: // %for.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp s2, s3, [x8], #8
+; CHECK-IADISABLED-NEXT: fcmp s2, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s2, gt
+; CHECK-IADISABLED-NEXT: fcmp s2, #0.0
+; CHECK-IADISABLED-NEXT: fcsel s2, s0, s4, mi
+; CHECK-IADISABLED-NEXT: fcmp s3, s1
+; CHECK-IADISABLED-NEXT: fcsel s4, s1, s3, gt
+; CHECK-IADISABLED-NEXT: fcmp s3, #0.0
+; CHECK-IADISABLED-NEXT: fcvtzs s2, s2
+; CHECK-IADISABLED-NEXT: fcsel s3, s0, s4, mi
+; CHECK-IADISABLED-NEXT: subs w10, w10, #1
+; CHECK-IADISABLED-NEXT: str b2, [x9]
+; CHECK-IADISABLED-NEXT: fcvtzs s3, s3
+; CHECK-IADISABLED-NEXT: stur b3, [x9, #1]
+; CHECK-IADISABLED-NEXT: add x9, x9, #2
+; CHECK-IADISABLED-NEXT: b.ne .LBB4_5
+; CHECK-IADISABLED-NEXT: .LBB4_6: // %for.cond.cleanup
+; CHECK-IADISABLED-NEXT: ret
+; CHECK-IADISABLED-NEXT: .LBB4_7: // %vector.ph
+; CHECK-IADISABLED-NEXT: add x11, x8, #1
+; CHECK-IADISABLED-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-IADISABLED-NEXT: and x10, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: dup v0.4s, w8
+; CHECK-IADISABLED-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-IADISABLED-NEXT: add x8, x1, x10, lsl #3
+; CHECK-IADISABLED-NEXT: add x9, x0, x10, lsl #1
+; CHECK-IADISABLED-NEXT: .LBB4_8: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [x1], #32
+; CHECK-IADISABLED-NEXT: subs x12, x12, #4
+; CHECK-IADISABLED-NEXT: uzp1 v3.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fcmgt v2.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fcmgt v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: bsl v2.16b, v0.16b, v3.16b
+; CHECK-IADISABLED-NEXT: fcmlt v3.4s, v3.4s, #0.0
+; CHECK-IADISABLED-NEXT: bsl v4.16b, v0.16b, v1.16b
+; CHECK-IADISABLED-NEXT: fcmlt v1.4s, v1.4s, #0.0
+; CHECK-IADISABLED-NEXT: bic v2.16b, v2.16b, v3.16b
+; CHECK-IADISABLED-NEXT: bic v1.16b, v4.16b, v1.16b
+; CHECK-IADISABLED-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: xtn v2.4h, v2.4s
+; CHECK-IADISABLED-NEXT: xtn v1.4h, v1.4s
+; CHECK-IADISABLED-NEXT: trn1 v1.8b, v2.8b, v1.8b
+; CHECK-IADISABLED-NEXT: str d1, [x0], #8
+; CHECK-IADISABLED-NEXT: b.ne .LBB4_8
+; CHECK-IADISABLED-NEXT: // %bb.9: // %middle.block
+; CHECK-IADISABLED-NEXT: cmp x11, x10
+; CHECK-IADISABLED-NEXT: b.ne .LBB4_4
+; CHECK-IADISABLED-NEXT: b .LBB4_6
+entry:
+ %cmp19 = icmp sgt i32 %width, 0
+ br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ %0 = add i32 %width, -1
+ %1 = zext i32 %0 to i64
+ %2 = add nuw nsw i64 %1, 1
+ %min.iters.check = icmp ult i32 %0, 3
+ br i1 %min.iters.check, label %for.body.preheader35, label %vector.memcheck
+
+vector.memcheck: ; preds = %for.body.preheader
+ %3 = add i32 %width, -1
+ %4 = zext i32 %3 to i64
+ %5 = shl nuw nsw i64 %4, 1
+ %6 = add nuw nsw i64 %5, 2
+ %scevgep = getelementptr i8, ptr %dst, i64 %6
+ %scevgep24 = getelementptr float, ptr %data, i64 %6
+ %bound0 = icmp ugt ptr %scevgep24, %dst
+ %bound1 = icmp ugt ptr %scevgep, %data
+ %found.conflict = and i1 %bound0, %bound1
+ br i1 %found.conflict, label %for.body.preheader35, label %vector.ph
+
+vector.ph: ; preds = %vector.memcheck
+ %n.vec = and i64 %2, 8589934588
+ %ind.end = trunc i64 %n.vec to i32
+ %7 = shl nuw nsw i64 %n.vec, 1
+ %ind.end27 = getelementptr float, ptr %data, i64 %7
+ %8 = shl nuw nsw i64 %n.vec, 1
+ %ind.end29 = getelementptr i8, ptr %dst, i64 %8
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %9 = shl i64 %index, 1
+ %next.gep = getelementptr float, ptr %data, i64 %9
+ %10 = shl i64 %index, 1
+ %wide.vec = load <8 x float>, ptr %next.gep, align 4
+ %deinterleaved.vec = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %strided.vec = extractvalue { <4 x float>, <4 x float> } %deinterleaved.vec, 0
+ %strided.vec34 = extractvalue { <4 x float>, <4 x float> } %deinterleaved.vec, 1
+ %11 = fcmp olt <4 x float> %strided.vec, zeroinitializer
+ %12 = fcmp ogt <4 x float> %strided.vec, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %13 = select <4 x i1> %12, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec
+ %14 = select <4 x i1> %11, <4 x float> zeroinitializer, <4 x float> %13
+ %15 = fptoui <4 x float> %14 to <4 x i8>
+ %16 = fcmp olt <4 x float> %strided.vec34, zeroinitializer
+ %17 = fcmp ogt <4 x float> %strided.vec34, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
+ %18 = select <4 x i1> %17, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec34
+ %19 = select <4 x i1> %16, <4 x float> zeroinitializer, <4 x float> %18
+ %20 = fptoui <4 x float> %19 to <4 x i8>
+ %21 = getelementptr inbounds i8, ptr %dst, i64 %10
+ %interleaved.vec = call <8 x i8> @llvm.vector.interleave2.v8i8(<4 x i8> %15, <4 x i8> %20)
+ store <8 x i8> %interleaved.vec, ptr %21, align 1
+ %index.next = add nuw i64 %index, 4
+ %22 = icmp eq i64 %index.next, %n.vec
+ br i1 %22, label %middle.block, label %vector.body
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %2, %n.vec
+ br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader35
+
+for.body.preheader35: ; preds = %vector.memcheck, %for.body.preheader, %middle.block
+ %i.022.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %ind.end, %middle.block ]
+ %src.021.ph = phi ptr [ %data, %vector.memcheck ], [ %data, %for.body.preheader ], [ %ind.end27, %middle.block ]
+ %dst.addr.020.ph = phi ptr [ %dst, %vector.memcheck ], [ %dst, %for.body.preheader ], [ %ind.end29, %middle.block ]
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader35, %for.body
+ %i.022 = phi i32 [ %inc, %for.body ], [ %i.022.ph, %for.body.preheader35 ]
+ %src.021 = phi ptr [ %add.ptr, %for.body ], [ %src.021.ph, %for.body.preheader35 ]
+ %dst.addr.020 = phi ptr [ %add.ptr6, %for.body ], [ %dst.addr.020.ph, %for.body.preheader35 ]
+ %23 = load float, ptr %src.021, align 4
+ %cmp.i = fcmp olt float %23, 0.000000e+00
+ %cmp1.i = fcmp ogt float %23, 2.550000e+02
+ %.x.i = select i1 %cmp1.i, float 2.550000e+02, float %23
+ %retval.0.i = select i1 %cmp.i, float 0.000000e+00, float %.x.i
+ %conv = fptoui float %retval.0.i to i8
+ store i8 %conv, ptr %dst.addr.020, align 1
+ %arrayidx2 = getelementptr inbounds float, ptr %src.021, i64 1
+ %24 = load float, ptr %arrayidx2, align 4
+ %cmp.i15 = fcmp olt float %24, 0.000000e+00
+ %cmp1.i16 = fcmp ogt float %24, 2.550000e+02
+ %.x.i17 = select i1 %cmp1.i16, float 2.550000e+02, float %24
+ %retval.0.i18 = select i1 %cmp.i15, float 0.000000e+00, float %.x.i17
+ %conv4 = fptoui float %retval.0.i18 to i8
+ %arrayidx5 = getelementptr inbounds i8, ptr %dst.addr.020, i64 1
+ store i8 %conv4, ptr %arrayidx5, align 1
+ %add.ptr = getelementptr inbounds float, ptr %src.021, i64 2
+ %add.ptr6 = getelementptr inbounds i8, ptr %dst.addr.020, i64 2
+ %inc = add nuw nsw i32 %i.022, 1
+ %exitcond.not = icmp eq i32 %inc, %width
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
+; CHECK-LABEL: loop3_intrinsic:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs w8, w2, #1
-; CHECK-NEXT: b.lt .LBB2_6
+; CHECK-NEXT: b.lt .LBB5_7
; CHECK-NEXT: // %bb.1: // %for.body.preheader
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: cmp w8, #2
-; CHECK-NEXT: b.ls .LBB2_3
+; CHECK-NEXT: b.ls .LBB5_3
; CHECK-NEXT: // %bb.2: // %vector.memcheck
; CHECK-NEXT: add x9, x8, w8, uxtw #1
; CHECK-NEXT: add x9, x9, #3
@@ -329,17 +1333,17 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: add x9, x1, x9, lsl #2
; CHECK-NEXT: cmp x10, x1
; CHECK-NEXT: ccmp x9, x0, #0, hi
-; CHECK-NEXT: b.ls .LBB2_7
-; CHECK-NEXT: .LBB2_3:
+; CHECK-NEXT: b.ls .LBB5_8
+; CHECK-NEXT: .LBB5_3:
; CHECK-NEXT: mov w10, wzr
; CHECK-NEXT: mov x8, x1
; CHECK-NEXT: mov x9, x0
-; CHECK-NEXT: .LBB2_4: // %for.body.preheader1
+; CHECK-NEXT: .LBB5_4: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
-; CHECK-NEXT: .LBB2_5: // %for.body
+; CHECK-NEXT: .LBB5_5: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp s2, s3, [x8]
; CHECK-NEXT: fcmp s2, s1
@@ -364,24 +1368,27 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: fcvtzs s3, s3
; CHECK-NEXT: stur b3, [x9, #2]
; CHECK-NEXT: add x9, x9, #3
-; CHECK-NEXT: b.ne .LBB2_5
-; CHECK-NEXT: .LBB2_6: // %for.cond.cleanup
+; CHECK-NEXT: b.ne .LBB5_5
+; CHECK-NEXT: .LBB5_6:
+; CHECK-NEXT: add sp, sp, #32
+; CHECK-NEXT: .LBB5_7: // %for.cond.cleanup
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB2_7: // %vector.ph
+; CHECK-NEXT: .LBB5_8: // %vector.ph
; CHECK-NEXT: add x11, x8, #1
; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-NEXT: adrp x12, .LCPI2_0
+; CHECK-NEXT: adrp x12, .LCPI5_0
; CHECK-NEXT: and x10, x11, #0x1fffffffc
; CHECK-NEXT: dup v0.4s, w8
-; CHECK-NEXT: ldr q1, [x12, :lo12:.LCPI2_0]
+; CHECK-NEXT: ldr q1, [x12, :lo12:.LCPI5_0]
; CHECK-NEXT: add x9, x10, x10, lsl #1
-; CHECK-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-NEXT: add x12, sp, #8
+; CHECK-NEXT: and x13, x11, #0x1fffffffc
; CHECK-NEXT: add x8, x1, x9, lsl #2
; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: .LBB2_8: // %vector.body
+; CHECK-NEXT: .LBB5_9: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ld3 { v2.4s, v3.4s, v4.4s }, [x1], #48
-; CHECK-NEXT: subs x12, x12, #4
+; CHECK-NEXT: subs x13, x13, #4
; CHECK-NEXT: fcmgt v5.4s, v2.4s, v0.4s
; CHECK-NEXT: fcmgt v6.4s, v3.4s, v0.4s
; CHECK-NEXT: fcmgt v7.4s, v4.4s, v0.4s
@@ -400,16 +1407,19 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: xtn v5.4h, v3.4s
; CHECK-NEXT: xtn v6.4h, v4.4s
; CHECK-NEXT: xtn v7.4h, v2.4s
-; CHECK-NEXT: tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
+; CHECK-NEXT: st3 { v5.4h, v6.4h, v7.4h }, [x12]
+; CHECK-NEXT: ldp d3, d4, [sp, #16]
+; CHECK-NEXT: ldr d2, [sp, #8]
+; CHECK-NEXT: tbl v2.16b, { v2.16b, v3.16b, v4.16b }, v1.16b
; CHECK-NEXT: mov s3, v2.s[2]
; CHECK-NEXT: str d2, [x0]
; CHECK-NEXT: str s3, [x0, #8]
; CHECK-NEXT: add x0, x0, #12
-; CHECK-NEXT: b.ne .LBB2_8
-; CHECK-NEXT: // %bb.9: // %middle.block
+; CHECK-NEXT: b.ne .LBB5_9
+; CHECK-NEXT: // %bb.10: // %middle.block
; CHECK-NEXT: cmp x11, x10
-; CHECK-NEXT: b.ne .LBB2_4
-; CHECK-NEXT: b .LBB2_6
+; CHECK-NEXT: b.ne .LBB5_4
+; CHECK-NEXT: b .LBB5_6
entry:
%cmp29 = icmp sgt i32 %width, 0
br i1 %cmp29, label %for.body.preheader, label %for.cond.cleanup
@@ -448,9 +1458,10 @@ vector.body: ; preds = %vector.body, %vecto
%next.gep = getelementptr float, ptr %data, i64 %9
%10 = mul i64 %index, 3
%wide.vec = load <12 x float>, ptr %next.gep, align 4
- %strided.vec = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %strided.vec44 = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %strided.vec45 = shufflevector <12 x float> %wide.vec, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %deinterleaved.vec = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide.vec)
+ %strided.vec = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 0
+ %strided.vec44 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 1
+ %strided.vec45 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 2
%11 = fcmp olt <4 x float> %strided.vec, zeroinitializer
%12 = fcmp ogt <4 x float> %strided.vec, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
%13 = select <4 x i1> %12, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec
@@ -467,9 +1478,7 @@ vector.body: ; preds = %vector.body, %vecto
%24 = select <4 x i1> %21, <4 x float> zeroinitializer, <4 x float> %23
%25 = fptoui <4 x float> %24 to <4 x i8>
%26 = getelementptr inbounds i8, ptr %dst, i64 %10
- %27 = shufflevector <4 x i8> %15, <4 x i8> %20, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %28 = shufflevector <4 x i8> %25, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
- %interleaved.vec = shufflevector <8 x i8> %27, <8 x i8> %28, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ %interleaved.vec = call <12 x i8> @llvm.vector.interleave3.v12i8(<4 x i8> %15, <4 x i8> %20, <4 x i8> %25)
store <12 x i8> %interleaved.vec, ptr %26, align 1
%index.next = add nuw i64 %index, 4
%29 = icmp eq i64 %index.next, %n.vec
@@ -524,14 +1533,15 @@ for.body: ; preds = %for.body.preheader4
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
-; CHECK-LABEL: loop4:
+
+define void @loop4_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
+; CHECK-LABEL: loop4_intrinsic:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs w8, w2, #1
-; CHECK-NEXT: b.lt .LBB3_6
+; CHECK-NEXT: b.lt .LBB6_6
; CHECK-NEXT: // %bb.1: // %for.body.preheader
; CHECK-NEXT: cmp w8, #2
-; CHECK-NEXT: b.ls .LBB3_3
+; CHECK-NEXT: b.ls .LBB6_3
; CHECK-NEXT: // %bb.2: // %vector.memcheck
; CHECK-NEXT: ubfiz x9, x8, #2, #32
; CHECK-NEXT: add x9, x9, #4
@@ -539,17 +1549,17 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: add x9, x1, x9, lsl #2
; CHECK-NEXT: cmp x10, x1
; CHECK-NEXT: ccmp x9, x0, #0, hi
-; CHECK-NEXT: b.ls .LBB3_7
-; CHECK-NEXT: .LBB3_3:
+; CHECK-NEXT: b.ls .LBB6_7
+; CHECK-NEXT: .LBB6_3:
; CHECK-NEXT: mov w10, wzr
; CHECK-NEXT: mov x8, x1
; CHECK-NEXT: mov x9, x0
-; CHECK-NEXT: .LBB3_4: // %for.body.preheader1
+; CHECK-NEXT: .LBB6_4: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
-; CHECK-NEXT: .LBB3_5: // %for.body
+; CHECK-NEXT: .LBB6_5: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp s2, s3, [x8]
; CHECK-NEXT: fcmp s2, s1
@@ -580,54 +1590,58 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: fcvtzs s5, s5
; CHECK-NEXT: stur b5, [x9, #3]
; CHECK-NEXT: add x9, x9, #4
-; CHECK-NEXT: b.ne .LBB3_5
-; CHECK-NEXT: .LBB3_6: // %for.cond.cleanup
+; CHECK-NEXT: b.ne .LBB6_5
+; CHECK-NEXT: .LBB6_6: // %for.cond.cleanup
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB3_7: // %vector.ph
+; CHECK-NEXT: .LBB6_7: // %vector.ph
; CHECK-NEXT: add x11, x8, #1
; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
-; CHECK-NEXT: adrp x12, .LCPI3_0
; CHECK-NEXT: and x10, x11, #0x1fffffffc
; CHECK-NEXT: dup v0.4s, w8
-; CHECK-NEXT: ldr q1, [x12, :lo12:.LCPI3_0]
+; CHECK-NEXT: and x12, x11, #0x1fffffffc
; CHECK-NEXT: add x8, x1, x10, lsl #4
; CHECK-NEXT: add x9, x0, x10, lsl #2
-; CHECK-NEXT: and x12, x11, #0x1fffffffc
-; CHECK-NEXT: .LBB3_8: // %vector.body
+; CHECK-NEXT: .LBB6_8: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld4 { v2.4s, v3.4s, v4.4s, v5.4s }, [x1], #64
+; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x1], #64
; CHECK-NEXT: subs x12, x12, #4
-; CHECK-NEXT: fcmgt v6.4s, v2.4s, v0.4s
-; CHECK-NEXT: fcmgt v7.4s, v3.4s, v0.4s
-; CHECK-NEXT: fcmgt v16.4s, v4.4s, v0.4s
-; CHECK-NEXT: fcmgt v17.4s, v5.4s, v0.4s
+; CHECK-NEXT: fcmgt v5.4s, v2.4s, v0.4s
+; CHECK-NEXT: fcmgt v6.4s, v4.4s, v0.4s
+; CHECK-NEXT: fcmgt v7.4s, v1.4s, v0.4s
+; CHECK-NEXT: fcmgt v16.4s, v3.4s, v0.4s
+; CHECK-NEXT: fcmlt v17.4s, v1.4s, #0.0
; CHECK-NEXT: fcmlt v18.4s, v2.4s, #0.0
-; CHECK-NEXT: fcmlt v19.4s, v3.4s, #0.0
-; CHECK-NEXT: fcmlt v20.4s, v4.4s, #0.0
-; CHECK-NEXT: bsl v6.16b, v0.16b, v2.16b
-; CHECK-NEXT: bsl v7.16b, v0.16b, v3.16b
-; CHECK-NEXT: bsl v16.16b, v0.16b, v4.16b
-; CHECK-NEXT: bsl v17.16b, v0.16b, v5.16b
-; CHECK-NEXT: fcmlt v2.4s, v5.4s, #0.0
-; CHECK-NEXT: bic v3.16b, v6.16b, v18.16b
-; CHECK-NEXT: bic v4.16b, v7.16b, v19.16b
-; CHECK-NEXT: bic v5.16b, v16.16b, v20.16b
-; CHECK-NEXT: bic v2.16b, v17.16b, v2.16b
+; CHECK-NEXT: fcmlt v19.4s, v4.4s, #0.0
+; CHECK-NEXT: bsl v5.16b, v0.16b, v2.16b
+; CHECK-NEXT: bsl v6.16b, v0.16b, v4.16b
+; CHECK-NEXT: bsl v7.16b, v0.16b, v1.16b
+; CHECK-NEXT: bsl v16.16b, v0.16b, v3.16b
+; CHECK-NEXT: fcmlt v1.4s, v3.4s, #0.0
+; CHECK-NEXT: bic v2.16b, v5.16b, v18.16b
+; CHECK-NEXT: bic v3.16b, v6.16b, v19.16b
+; CHECK-NEXT: bic v4.16b, v7.16b, v17.16b
+; CHECK-NEXT: bic v1.16b, v16.16b, v1.16b
+; CHECK-NEXT: fcvtzs v2.4s, v2.4s
; CHECK-NEXT: fcvtzs v3.4s, v3.4s
; CHECK-NEXT: fcvtzs v4.4s, v4.4s
-; CHECK-NEXT: fcvtzs v5.4s, v5.4s
-; CHECK-NEXT: fcvtzs v2.4s, v2.4s
-; CHECK-NEXT: xtn v16.4h, v3.4s
-; CHECK-NEXT: xtn v17.4h, v4.4s
-; CHECK-NEXT: xtn v18.4h, v5.4s
-; CHECK-NEXT: xtn v19.4h, v2.4s
-; CHECK-NEXT: tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
-; CHECK-NEXT: str q2, [x0], #16
-; CHECK-NEXT: b.ne .LBB3_8
+; CHECK-NEXT: fcvtzs v1.4s, v1.4s
+; CHECK-NEXT: xtn v2.4h, v2.4s
+; CHECK-NEXT: xtn v3.4h, v3.4s
+; CHECK-NEXT: xtn v4.4h, v4.4s
+; CHECK-NEXT: xtn v1.4h, v1.4s
+; CHECK-NEXT: zip2 v5.4h, v2.4h, v3.4h
+; CHECK-NEXT: zip1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT: zip2 v6.4h, v4.4h, v1.4h
+; CHECK-NEXT: zip1 v1.4h, v4.4h, v1.4h
+; CHECK-NEXT: zip1 v3.8h, v6.8h, v5.8h
+; CHECK-NEXT: zip1 v1.8h, v1.8h, v2.8h
+; CHECK-NEXT: uzp1 v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: str q1, [x0], #16
+; CHECK-NEXT: b.ne .LBB6_8
; CHECK-NEXT: // %bb.9: // %middle.block
; CHECK-NEXT: cmp x11, x10
-; CHECK-NEXT: b.ne .LBB3_4
-; CHECK-NEXT: b .LBB3_6
+; CHECK-NEXT: b.ne .LBB6_4
+; CHECK-NEXT: b .LBB6_6
entry:
%cmp39 = icmp sgt i32 %width, 0
br i1 %cmp39, label %for.body.preheader, label %for.cond.cleanup
@@ -666,10 +1680,11 @@ vector.body: ; preds = %vector.body, %vecto
%next.gep = getelementptr float, ptr %data, i64 %9
%10 = shl i64 %index, 2
%wide.vec = load <16 x float>, ptr %next.gep, align 4
- %strided.vec = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %strided.vec54 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %strided.vec55 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %strided.vec56 = shufflevector <16 x float> %wide.vec, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %deinterleaved.vec = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %strided.vec = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 0
+ %strided.vec54 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 1
+ %strided.vec55 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 2
+ %strided.vec56 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleaved.vec, 3
%11 = fcmp olt <4 x float> %strided.vec, zeroinitializer
%12 = fcmp ogt <4 x float> %strided.vec, <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>
%13 = select <4 x i1> %12, <4 x float> <float 2.550000e+02, float 2.550000e+02, float 2.550000e+02, float 2.550000e+02>, <4 x float> %strided.vec
@@ -691,9 +1706,7 @@ vector.body: ; preds = %vector.body, %vecto
%29 = select <4 x i1> %26, <4 x float> zeroinitializer, <4 x float> %28
%30 = fptoui <4 x float> %29 to <4 x i8>
%31 = getelementptr inbounds i8, ptr %dst, i64 %10
- %32 = shufflevector <4 x i8> %15, <4 x i8> %20, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %33 = shufflevector <4 x i8> %25, <4 x i8> %30, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %interleaved.vec = shufflevector <8 x i8> %32, <8 x i8> %33, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ %interleaved.vec = call <16 x i8> @llvm.vector.interleave4.v16i8(<4 x i8> %15, <4 x i8> %20, <4 x i8> %25, <4 x i8> %30)
store <16 x i8> %interleaved.vec, ptr %31, align 1
%index.next = add nuw i64 %index, 4
%34 = icmp eq i64 %index.next, %n.vec
diff --git a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
index 2bf0a679bcd26..0ffdf4eedbaaf 100644
--- a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
@@ -1,23 +1,42 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
define void @vld2(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld2:
-; CHECK: .Lfunc_begin0:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB0_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
-; CHECK-NEXT: fmul v2.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v2.4s, v1.4s, v1.4s
-; CHECK-NEXT: str q2, [x1, x8]
-; CHECK-NEXT: add x8, x8, #16
-; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-NEXT: b.ne .LBB0_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld2:
+; CHECK-IAENABLED: .Lfunc_begin0:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB0_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: fmul v2.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v2.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: str q2, [x1, x8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IAENABLED-NEXT: b.ne .LBB0_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2:
+; CHECK-IADISABLED: .Lfunc_begin0:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB0_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0], #32
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q0, [x1, x8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IADISABLED-NEXT: b.ne .LBB0_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -42,23 +61,55 @@ while.end: ; preds = %vector.body
}
define void @vld3(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld3:
-; CHECK: .Lfunc_begin1:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB1_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-; CHECK-NEXT: fmul v3.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v3.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v3.4s, v2.4s, v2.4s
-; CHECK-NEXT: str q3, [x1, x8]
-; CHECK-NEXT: add x8, x8, #16
-; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-NEXT: b.ne .LBB1_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld3:
+; CHECK-IAENABLED: .Lfunc_begin1:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB1_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-IAENABLED-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: str q3, [x1, x8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IAENABLED-NEXT: b.ne .LBB1_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld3:
+; CHECK-IADISABLED: .Lfunc_begin1:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB1_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: add x0, x0, #48
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q0, [x1, x8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IADISABLED-NEXT: b.ne .LBB1_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -86,25 +137,64 @@ while.end: ; preds = %vector.body
}
define void @vld4(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld4:
-; CHECK: .Lfunc_begin2:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB2_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-NEXT: add x9, x1, x8
-; CHECK-NEXT: add x8, x8, #32
-; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x9]
-; CHECK-NEXT: b.ne .LBB2_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld4:
+; CHECK-IAENABLED: .Lfunc_begin2:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB2_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT: add x9, x1, x8
+; CHECK-IAENABLED-NEXT: add x8, x8, #32
+; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-IAENABLED-NEXT: b.ne .LBB2_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4:
+; CHECK-IADISABLED: .Lfunc_begin2:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB2_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: add x9, x1, x8
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: add x8, x8, #32
+; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IADISABLED-NEXT: add x0, x0, #64
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v3.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmul v2.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v6.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: trn2 v7.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: trn1 v4.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: ext v2.16b, v2.16b, v5.16b, #8
+; CHECK-IADISABLED-NEXT: mov v7.d[1], v5.d[1]
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v4.d[1]
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v3.d[1]
+; CHECK-IADISABLED-NEXT: mov v6.d[1], v2.d[1]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fadd v2.4s, v7.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x9]
+; CHECK-IADISABLED-NEXT: b.ne .LBB2_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -136,25 +226,46 @@ while.end: ; preds = %vector.body
}
define void @twosrc(ptr nocapture readonly %pSrc, ptr nocapture readonly %pSrc2, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: twosrc:
-; CHECK: .Lfunc_begin3:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB3_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x10, x1, x8
-; CHECK-NEXT: add x8, x8, #32
-; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x9]
-; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-NEXT: ld2 { v2.4s, v3.4s }, [x10]
-; CHECK-NEXT: fmul v4.4s, v2.4s, v0.4s
-; CHECK-NEXT: fmla v4.4s, v1.4s, v3.4s
-; CHECK-NEXT: str q4, [x2], #16
-; CHECK-NEXT: b.ne .LBB3_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: twosrc:
+; CHECK-IAENABLED: .Lfunc_begin3:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB3_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: add x9, x0, x8
+; CHECK-IAENABLED-NEXT: add x10, x1, x8
+; CHECK-IAENABLED-NEXT: add x8, x8, #32
+; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x9]
+; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IAENABLED-NEXT: ld2 { v2.4s, v3.4s }, [x10]
+; CHECK-IAENABLED-NEXT: fmul v4.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v3.4s
+; CHECK-IAENABLED-NEXT: str q4, [x2], #16
+; CHECK-IAENABLED-NEXT: b.ne .LBB3_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc:
+; CHECK-IADISABLED: .Lfunc_begin3:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB3_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: add x9, x0, x8
+; CHECK-IADISABLED-NEXT: add x10, x1, x8
+; CHECK-IADISABLED-NEXT: add x8, x8, #32
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x9]
+; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x10]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q0, [x2], #16
+; CHECK-IADISABLED-NEXT: b.ne .LBB3_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -182,22 +293,40 @@ while.end: ; preds = %vector.body
}
define void @vld2_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld2_multiuse:
-; CHECK: .Lfunc_begin4:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB4_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
-; CHECK-NEXT: fmul v2.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v2.4s, v1.4s, v1.4s
-; CHECK-NEXT: str q2, [x1, x8]
-; CHECK-NEXT: add x8, x8, #16
-; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-NEXT: b.ne .LBB4_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld2_multiuse:
+; CHECK-IAENABLED: .Lfunc_begin4:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB4_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: fmul v2.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v2.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: str q2, [x1, x8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IAENABLED-NEXT: b.ne .LBB4_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2_multiuse:
+; CHECK-IADISABLED: .Lfunc_begin4:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB4_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0], #32
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q0, [x1, x8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IADISABLED-NEXT: b.ne .LBB4_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -221,23 +350,55 @@ while.end: ; preds = %vector.body
}
define void @vld3_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld3_multiuse:
-; CHECK: .Lfunc_begin5:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB5_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-; CHECK-NEXT: fmul v3.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v3.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v3.4s, v2.4s, v2.4s
-; CHECK-NEXT: str q3, [x1, x8]
-; CHECK-NEXT: add x8, x8, #16
-; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
-; CHECK-NEXT: b.ne .LBB5_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld3_multiuse:
+; CHECK-IAENABLED: .Lfunc_begin5:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB5_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-IAENABLED-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: str q3, [x1, x8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IAENABLED-NEXT: b.ne .LBB5_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld3_multiuse:
+; CHECK-IADISABLED: .Lfunc_begin5:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB5_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: add x0, x0, #48
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q0, [x1, x8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IADISABLED-NEXT: b.ne .LBB5_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -263,25 +424,64 @@ while.end: ; preds = %vector.body
}
define void @vld4_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
-; CHECK-LABEL: vld4_multiuse:
-; CHECK: .Lfunc_begin6:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB6_1: // %vector.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
-; CHECK-NEXT: add x9, x1, x8
-; CHECK-NEXT: add x8, x8, #32
-; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
-; CHECK-NEXT: fmul v4.4s, v0.4s, v0.4s
-; CHECK-NEXT: fmla v4.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmul v5.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v5.4s, v3.4s, v3.4s
-; CHECK-NEXT: st2 { v4.4s, v5.4s }, [x9]
-; CHECK-NEXT: b.ne .LBB6_1
-; CHECK-NEXT: // %bb.2: // %while.end
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld4_multiuse:
+; CHECK-IAENABLED: .Lfunc_begin6:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB6_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT: add x9, x1, x8
+; CHECK-IAENABLED-NEXT: add x8, x8, #32
+; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-IAENABLED-NEXT: b.ne .LBB6_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4_multiuse:
+; CHECK-IADISABLED: .Lfunc_begin6:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB6_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: add x9, x1, x8
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: add x8, x8, #32
+; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IADISABLED-NEXT: add x0, x0, #64
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v3.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmul v2.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v6.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: trn2 v7.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: trn1 v4.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: ext v2.16b, v2.16b, v5.16b, #8
+; CHECK-IADISABLED-NEXT: mov v7.d[1], v5.d[1]
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v4.d[1]
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v3.d[1]
+; CHECK-IADISABLED-NEXT: mov v6.d[1], v2.d[1]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fadd v2.4s, v7.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x9]
+; CHECK-IADISABLED-NEXT: b.ne .LBB6_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
entry:
br label %vector.body
@@ -312,24 +512,44 @@ while.end: ; preds = %vector.body
; This example has store(shuffle(shuffle(... that would be better to be treated
; as a single store. This avoids the vld2 for data that is already shuffled.
define void @transpose_s16_8x8_simpler(ptr nocapture noundef %a) {
-; CHECK-LABEL: transpose_s16_8x8_simpler:
-; CHECK: .Lfunc_begin7:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ldp q2, q3, [x0, #64]
-; CHECK-NEXT: ldp q4, q5, [x0, #32]
-; CHECK-NEXT: ldp q6, q7, [x0, #96]
-; CHECK-NEXT: trn1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: trn1 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT: trn1 v2.8h, v4.8h, v5.8h
-; CHECK-NEXT: trn1 v3.8h, v6.8h, v7.8h
-; CHECK-NEXT: trn1 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: trn1 v1.4s, v2.4s, v3.4s
-; CHECK-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: st2 { v0.2s, v1.2s }, [x0]
-; CHECK-NEXT: str q2, [x0, #64]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: transpose_s16_8x8_simpler:
+; CHECK-IAENABLED: .Lfunc_begin7:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IAENABLED-NEXT: ldp q2, q3, [x0, #64]
+; CHECK-IAENABLED-NEXT: ldp q4, q5, [x0, #32]
+; CHECK-IAENABLED-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-IAENABLED-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-IAENABLED-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-IAENABLED-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-IAENABLED-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-IAENABLED-NEXT: trn1 v0.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v1.4s, v2.4s, v3.4s
+; CHECK-IAENABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x0]
+; CHECK-IAENABLED-NEXT: str q2, [x0, #64]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: transpose_s16_8x8_simpler:
+; CHECK-IADISABLED: .Lfunc_begin7:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-IADISABLED-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-IADISABLED-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-IADISABLED-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-IADISABLED-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-IADISABLED-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: trn1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q2, [x0]
+; CHECK-IADISABLED-NEXT: str q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -365,24 +585,44 @@ entry:
; Same as above with some different shuffles
define void @transpose_s16_8x8_simpler2(ptr nocapture noundef %a) {
-; CHECK-LABEL: transpose_s16_8x8_simpler2:
-; CHECK: .Lfunc_begin8:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ldp q2, q3, [x0, #64]
-; CHECK-NEXT: ldp q4, q5, [x0, #96]
-; CHECK-NEXT: trn1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: zip1 v1.8h, v2.8h, v3.8h
-; CHECK-NEXT: ldp q2, q3, [x0, #32]
-; CHECK-NEXT: trn1 v4.8h, v4.8h, v5.8h
-; CHECK-NEXT: zip1 v3.8h, v2.8h, v3.8h
-; CHECK-NEXT: trn1 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: dup v0.4s, v4.s[2]
-; CHECK-NEXT: zip2 v0.4s, v2.4s, v0.4s
-; CHECK-NEXT: st2 { v2.2s, v3.2s }, [x0]
-; CHECK-NEXT: str q0, [x0, #64]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: transpose_s16_8x8_simpler2:
+; CHECK-IAENABLED: .Lfunc_begin8:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IAENABLED-NEXT: ldp q2, q3, [x0, #64]
+; CHECK-IAENABLED-NEXT: ldp q4, q5, [x0, #96]
+; CHECK-IAENABLED-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-IAENABLED-NEXT: zip1 v1.8h, v2.8h, v3.8h
+; CHECK-IAENABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IAENABLED-NEXT: trn1 v4.8h, v4.8h, v5.8h
+; CHECK-IAENABLED-NEXT: zip1 v3.8h, v2.8h, v3.8h
+; CHECK-IAENABLED-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: dup v0.4s, v4.s[2]
+; CHECK-IAENABLED-NEXT: zip2 v0.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x0]
+; CHECK-IAENABLED-NEXT: str q0, [x0, #64]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: transpose_s16_8x8_simpler2:
+; CHECK-IADISABLED: .Lfunc_begin8:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-IADISABLED-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-IADISABLED-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-IADISABLED-NEXT: zip1 v1.8h, v2.8h, v3.8h
+; CHECK-IADISABLED-NEXT: zip1 v2.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-IADISABLED-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: str q2, [x0]
+; CHECK-IADISABLED-NEXT: str q0, [x0, #64]
+; CHECK-IADISABLED-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -418,47 +658,93 @@ entry:
define void @transpose_s16_8x8(ptr nocapture noundef %0, ptr nocapture noundef %1, ptr nocapture noundef %2, ptr nocapture noundef %3, ptr nocapture noundef %4, ptr nocapture noundef %5, ptr nocapture noundef %6, ptr nocapture noundef %7) {
-; CHECK-LABEL: transpose_s16_8x8:
-; CHECK: .Lfunc_begin9:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: ldr q3, [x4]
-; CHECK-NEXT: ldr q4, [x5]
-; CHECK-NEXT: ldr q2, [x2]
-; CHECK-NEXT: ldr q5, [x3]
-; CHECK-NEXT: trn1 v16.8h, v0.8h, v1.8h
-; CHECK-NEXT: trn2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: ldr q6, [x6]
-; CHECK-NEXT: ldr q7, [x7]
-; CHECK-NEXT: trn1 v17.8h, v3.8h, v4.8h
-; CHECK-NEXT: trn2 v1.8h, v3.8h, v4.8h
-; CHECK-NEXT: trn1 v18.8h, v2.8h, v5.8h
-; CHECK-NEXT: trn2 v2.8h, v2.8h, v5.8h
-; CHECK-NEXT: trn1 v19.8h, v6.8h, v7.8h
-; CHECK-NEXT: trn2 v3.8h, v6.8h, v7.8h
-; CHECK-NEXT: trn1 v4.4s, v16.4s, v17.4s
-; CHECK-NEXT: trn1 v6.4s, v0.4s, v1.4s
-; CHECK-NEXT: trn2 v16.4s, v16.4s, v17.4s
-; CHECK-NEXT: trn2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: trn1 v5.4s, v18.4s, v19.4s
-; CHECK-NEXT: trn1 v7.4s, v2.4s, v3.4s
-; CHECK-NEXT: trn2 v17.4s, v18.4s, v19.4s
-; CHECK-NEXT: trn2 v1.4s, v2.4s, v3.4s
-; CHECK-NEXT: st2 { v4.2s, v5.2s }, [x0]
-; CHECK-NEXT: zip2 v2.4s, v4.4s, v5.4s
-; CHECK-NEXT: zip2 v3.4s, v6.4s, v7.4s
-; CHECK-NEXT: zip2 v4.4s, v16.4s, v17.4s
-; CHECK-NEXT: st2 { v6.2s, v7.2s }, [x1]
-; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x2]
-; CHECK-NEXT: st2 { v0.2s, v1.2s }, [x3]
-; CHECK-NEXT: zip2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: str q2, [x4]
-; CHECK-NEXT: str q3, [x5]
-; CHECK-NEXT: str q4, [x6]
-; CHECK-NEXT: str q0, [x7]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: transpose_s16_8x8:
+; CHECK-IAENABLED: .Lfunc_begin9:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0:
+; CHECK-IAENABLED-NEXT: ldr q0, [x0]
+; CHECK-IAENABLED-NEXT: ldr q1, [x1]
+; CHECK-IAENABLED-NEXT: ldr q3, [x4]
+; CHECK-IAENABLED-NEXT: ldr q4, [x5]
+; CHECK-IAENABLED-NEXT: ldr q2, [x2]
+; CHECK-IAENABLED-NEXT: ldr q5, [x3]
+; CHECK-IAENABLED-NEXT: trn1 v16.8h, v0.8h, v1.8h
+; CHECK-IAENABLED-NEXT: trn2 v0.8h, v0.8h, v1.8h
+; CHECK-IAENABLED-NEXT: ldr q6, [x6]
+; CHECK-IAENABLED-NEXT: ldr q7, [x7]
+; CHECK-IAENABLED-NEXT: trn1 v17.8h, v3.8h, v4.8h
+; CHECK-IAENABLED-NEXT: trn2 v1.8h, v3.8h, v4.8h
+; CHECK-IAENABLED-NEXT: trn1 v18.8h, v2.8h, v5.8h
+; CHECK-IAENABLED-NEXT: trn2 v2.8h, v2.8h, v5.8h
+; CHECK-IAENABLED-NEXT: trn1 v19.8h, v6.8h, v7.8h
+; CHECK-IAENABLED-NEXT: trn2 v3.8h, v6.8h, v7.8h
+; CHECK-IAENABLED-NEXT: trn1 v4.4s, v16.4s, v17.4s
+; CHECK-IAENABLED-NEXT: trn1 v6.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-IAENABLED-NEXT: trn2 v0.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v5.4s, v18.4s, v19.4s
+; CHECK-IAENABLED-NEXT: trn1 v7.4s, v2.4s, v3.4s
+; CHECK-IAENABLED-NEXT: trn2 v17.4s, v18.4s, v19.4s
+; CHECK-IAENABLED-NEXT: trn2 v1.4s, v2.4s, v3.4s
+; CHECK-IAENABLED-NEXT: st2 { v4.2s, v5.2s }, [x0]
+; CHECK-IAENABLED-NEXT: zip2 v2.4s, v4.4s, v5.4s
+; CHECK-IAENABLED-NEXT: zip2 v3.4s, v6.4s, v7.4s
+; CHECK-IAENABLED-NEXT: zip2 v4.4s, v16.4s, v17.4s
+; CHECK-IAENABLED-NEXT: st2 { v6.2s, v7.2s }, [x1]
+; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x2]
+; CHECK-IAENABLED-NEXT: st2 { v0.2s, v1.2s }, [x3]
+; CHECK-IAENABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: str q2, [x4]
+; CHECK-IAENABLED-NEXT: str q3, [x5]
+; CHECK-IAENABLED-NEXT: str q4, [x6]
+; CHECK-IAENABLED-NEXT: str q0, [x7]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: transpose_s16_8x8:
+; CHECK-IADISABLED: .Lfunc_begin9:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr q0, [x2]
+; CHECK-IADISABLED-NEXT: ldr q1, [x0]
+; CHECK-IADISABLED-NEXT: ldr q2, [x1]
+; CHECK-IADISABLED-NEXT: ldr q3, [x3]
+; CHECK-IADISABLED-NEXT: ldr q4, [x4]
+; CHECK-IADISABLED-NEXT: ldr q5, [x5]
+; CHECK-IADISABLED-NEXT: trn1 v16.8h, v1.8h, v2.8h
+; CHECK-IADISABLED-NEXT: trn2 v1.8h, v1.8h, v2.8h
+; CHECK-IADISABLED-NEXT: ldr q6, [x6]
+; CHECK-IADISABLED-NEXT: ldr q7, [x7]
+; CHECK-IADISABLED-NEXT: trn1 v2.8h, v0.8h, v3.8h
+; CHECK-IADISABLED-NEXT: trn1 v17.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: trn2 v0.8h, v0.8h, v3.8h
+; CHECK-IADISABLED-NEXT: trn2 v3.8h, v4.8h, v5.8h
+; CHECK-IADISABLED-NEXT: trn1 v18.8h, v6.8h, v7.8h
+; CHECK-IADISABLED-NEXT: trn2 v4.8h, v6.8h, v7.8h
+; CHECK-IADISABLED-NEXT: trn1 v5.4s, v16.4s, v17.4s
+; CHECK-IADISABLED-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-IADISABLED-NEXT: trn1 v7.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: trn2 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v18.4s
+; CHECK-IADISABLED-NEXT: trn1 v19.4s, v0.4s, v4.4s
+; CHECK-IADISABLED-NEXT: trn2 v2.4s, v2.4s, v18.4s
+; CHECK-IADISABLED-NEXT: trn2 v0.4s, v0.4s, v4.4s
+; CHECK-IADISABLED-NEXT: zip1 v3.4s, v5.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v7.4s, v19.4s
+; CHECK-IADISABLED-NEXT: zip1 v17.4s, v16.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v18.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip2 v5.4s, v5.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v16.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: str q3, [x0]
+; CHECK-IADISABLED-NEXT: zip2 v3.4s, v7.4s, v19.4s
+; CHECK-IADISABLED-NEXT: str q4, [x1]
+; CHECK-IADISABLED-NEXT: str q17, [x2]
+; CHECK-IADISABLED-NEXT: str q18, [x3]
+; CHECK-IADISABLED-NEXT: str q5, [x4]
+; CHECK-IADISABLED-NEXT: str q3, [x5]
+; CHECK-IADISABLED-NEXT: str q2, [x6]
+; CHECK-IADISABLED-NEXT: str q0, [x7]
+; CHECK-IADISABLED-NEXT: ret
%9 = load <8 x i16>, ptr %0, align 16
%10 = load <8 x i16>, ptr %1, align 16
%11 = shufflevector <8 x i16> %9, <8 x i16> %10, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
@@ -607,14 +893,25 @@ define void @transpose_s16_8x8_(ptr nocapture noundef %0) {
}
define void @store_factor2(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: store_factor2:
-; CHECK: .Lfunc_begin11:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: trn1 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: trn1 v3.4s, v1.4s, v0.4s
-; CHECK-NEXT: st2 { v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor2:
+; CHECK-IAENABLED: .Lfunc_begin11:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0:
+; CHECK-IAENABLED-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor2:
+; CHECK-IADISABLED: .Lfunc_begin11:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0:
+; CHECK-IADISABLED-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: trn1 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -663,21 +960,38 @@ define void @store_factor2_high2(ptr %ptr, ptr %ptr2, <4 x i32> %a0, <4 x i32> %
}
define void @store_factor3(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
-; CHECK-LABEL: store_factor3:
-; CHECK: .Lfunc_begin14:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ext v3.16b, v0.16b, v1.16b, #12
-; CHECK-NEXT: ext v6.16b, v1.16b, v2.16b, #12
-; CHECK-NEXT: zip2 v3.4s, v0.4s, v3.4s
-; CHECK-NEXT: mov v3.s[0], v0.s[0]
-; CHECK-NEXT: ext v0.16b, v2.16b, v0.16b, #12
-; CHECK-NEXT: zip2 v4.4s, v1.4s, v6.4s
-; CHECK-NEXT: mov v4.s[0], v1.s[0]
-; CHECK-NEXT: zip2 v5.4s, v2.4s, v0.4s
-; CHECK-NEXT: mov v5.s[0], v2.s[0]
-; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor3:
+; CHECK-IAENABLED: .Lfunc_begin14:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0:
+; CHECK-IAENABLED-NEXT: ext v3.16b, v0.16b, v1.16b, #12
+; CHECK-IAENABLED-NEXT: ext v6.16b, v1.16b, v2.16b, #12
+; CHECK-IAENABLED-NEXT: zip2 v3.4s, v0.4s, v3.4s
+; CHECK-IAENABLED-NEXT: mov v3.s[0], v0.s[0]
+; CHECK-IAENABLED-NEXT: ext v0.16b, v2.16b, v0.16b, #12
+; CHECK-IAENABLED-NEXT: zip2 v4.4s, v1.4s, v6.4s
+; CHECK-IAENABLED-NEXT: mov v4.s[0], v1.s[0]
+; CHECK-IAENABLED-NEXT: zip2 v5.4s, v2.4s, v0.4s
+; CHECK-IAENABLED-NEXT: mov v5.s[0], v2.s[0]
+; CHECK-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor3:
+; CHECK-IADISABLED: .Lfunc_begin14:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0:
+; CHECK-IADISABLED-NEXT: trn2 v3.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-IADISABLED-NEXT: ext v4.16b, v4.16b, v4.16b, #12
+; CHECK-IADISABLED-NEXT: mov v1.s[2], v2.s[0]
+; CHECK-IADISABLED-NEXT: uzp2 v5.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: mov v4.s[3], v0.s[2]
+; CHECK-IADISABLED-NEXT: str q1, [x0]
+; CHECK-IADISABLED-NEXT: ext v0.16b, v5.16b, v3.16b, #4
+; CHECK-IADISABLED-NEXT: stp q0, q4, [x0, #16]
+; CHECK-IADISABLED-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a0, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
@@ -689,16 +1003,38 @@ define void @store_factor3(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2
}
define void @store_factor4(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x i32> %a3) {
-; CHECK-LABEL: store_factor4:
-; CHECK: .Lfunc_begin15:
-; CHECK-NEXT: .cfi_startproc
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: trn1 v4.4s, v0.4s, v1.4s
-; CHECK-NEXT: trn1 v5.4s, v1.4s, v2.4s
-; CHECK-NEXT: trn1 v6.4s, v2.4s, v3.4s
-; CHECK-NEXT: trn1 v7.4s, v3.4s, v0.4s
-; CHECK-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor4:
+; CHECK-IAENABLED: .Lfunc_begin15:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0:
+; CHECK-IAENABLED-NEXT: trn1 v4.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v5.4s, v1.4s, v2.4s
+; CHECK-IAENABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-IAENABLED-NEXT: trn1 v7.4s, v3.4s, v0.4s
+; CHECK-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor4:
+; CHECK-IADISABLED: .Lfunc_begin15:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0:
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: trn1 v6.4s, v3.4s, v0.4s
+; CHECK-IADISABLED-NEXT: zip2 v7.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: trn1 v16.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v17.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ext v3.16b, v3.16b, v4.16b, #8
+; CHECK-IADISABLED-NEXT: zip1 v4.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; CHECK-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; CHECK-IADISABLED-NEXT: mov v17.d[1], v16.d[1]
+; CHECK-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-IADISABLED-NEXT: stp q17, q7, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q4, [x0]
+; CHECK-IADISABLED-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a3, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -730,7 +1066,695 @@ entry:
ret void
}
-declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+define void @vld2_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-LABEL: vld2_intrinsic:
+; CHECK: .Lfunc_begin17:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB17_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldp q1, q0, [x0], #32
+; CHECK-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v1.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v1.4s, v0.4s, v0.4s
+; CHECK-NEXT: str q1, [x1, x8]
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-NEXT: b.ne .LBB17_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = shl i64 %index, 1
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %next.gep19 = getelementptr float, ptr %pDst, i64 %index
+ %wide.vec = load <8 x float>, ptr %next.gep, align 4
+ %1 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %2 = extractvalue { <4 x float>, <4 x float> } %1, 0
+ %3 = fmul fast <4 x float> %2, %2
+ %4 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %5 = extractvalue { <4 x float>, <4 x float> } %4, 1
+ %6 = fmul fast <4 x float> %5, %5
+ %7 = fadd fast <4 x float> %6, %3
+ store <4 x float> %7, ptr %next.gep19, align 4
+ %index.next = add i64 %index, 4
+ %8 = icmp eq i64 %index.next, 1024
+ br i1 %8, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @vld3_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-LABEL: vld3_intrinsic:
+; CHECK: .Lfunc_begin18:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB18_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-NEXT: str q3, [x1, x8]
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-NEXT: b.ne .LBB18_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = mul i64 %index, 3
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %next.gep23 = getelementptr float, ptr %pDst, i64 %index
+ %wide.vec = load <12 x float>, ptr %next.gep, align 4
+ %1 = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide.vec)
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %1, 0
+ %3 = fmul fast <4 x float> %2, %2
+ %4 = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide.vec)
+ %5 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %4, 1
+ %6 = fmul fast <4 x float> %5, %5
+ %7 = fadd fast <4 x float> %6, %3
+ %8 = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide.vec)
+ %9 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %8, 2
+ %10 = fmul fast <4 x float> %9, %9
+ %11 = fadd fast <4 x float> %7, %10
+ store <4 x float> %11, ptr %next.gep23, align 4
+ %index.next = add i64 %index, 4
+ %12 = icmp eq i64 %index.next, 1024
+ br i1 %12, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @vld4_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-IAENABLED-LABEL: vld4_intrinsic:
+; CHECK-IAENABLED: .Lfunc_begin19:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB19_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT: add x9, x1, x8
+; CHECK-IAENABLED-NEXT: add x8, x8, #32
+; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-IAENABLED-NEXT: b.ne .LBB19_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4_intrinsic:
+; CHECK-IADISABLED: .Lfunc_begin19:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB19_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IADISABLED-NEXT: add x9, x1, x8
+; CHECK-IADISABLED-NEXT: add x8, x8, #32
+; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x9]
+; CHECK-IADISABLED-NEXT: b.ne .LBB19_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = shl i64 %index, 2
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %1 = shl i64 %index, 1
+ %wide.vec = load <16 x float>, ptr %next.gep, align 4
+ %2 = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %2, 0
+ %4 = fmul fast <4 x float> %3, %3
+ %5 = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %6 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %5, 1
+ %7 = fmul fast <4 x float> %6, %6
+ %8 = fadd fast <4 x float> %7, %4
+ %9 = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %10 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %9, 2
+ %11 = fmul fast <4 x float> %10, %10
+ %12 = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %13 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %12, 3
+ %14 = fmul fast <4 x float> %13, %13
+ %15 = fadd fast <4 x float> %14, %11
+ %16 = getelementptr inbounds float, ptr %pDst, i64 %1
+ %interleaved.vec = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %8, <4 x float> %15)
+ store <8 x float> %interleaved.vec, ptr %16, align 4
+ %index.next = add i64 %index, 4
+ %17 = icmp eq i64 %index.next, 1024
+ br i1 %17, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @twosrc_intrinsic(ptr nocapture readonly %pSrc, ptr nocapture readonly %pSrc2, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-LABEL: twosrc_intrinsic:
+; CHECK: .Lfunc_begin20:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB20_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x10, x1, x8
+; CHECK-NEXT: add x8, x8, #32
+; CHECK-NEXT: ldp q1, q0, [x9]
+; CHECK-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-NEXT: ldp q3, q2, [x10]
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-NEXT: uzp2 v1.4s, v3.4s, v2.4s
+; CHECK-NEXT: fmul v2.4s, v5.4s, v4.4s
+; CHECK-NEXT: fmla v2.4s, v0.4s, v1.4s
+; CHECK-NEXT: str q2, [x2], #16
+; CHECK-NEXT: b.ne .LBB20_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = shl i64 %index, 1
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %1 = shl i64 %index, 1
+ %next.gep23 = getelementptr float, ptr %pSrc2, i64 %1
+ %next.gep24 = getelementptr float, ptr %pDst, i64 %index
+ %wide.vec = load <8 x float>, ptr %next.gep, align 4
+ %wide.vec26 = load <8 x float>, ptr %next.gep23, align 4
+ %2 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %3 = extractvalue { <4 x float>, <4 x float> } %2, 0
+ %4 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec26)
+ %5 = extractvalue { <4 x float>, <4 x float> } %4, 0
+ %6 = fmul fast <4 x float> %5, %3
+ %7 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %8 = extractvalue { <4 x float>, <4 x float> } %7, 1
+ %9 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec26)
+ %10 = extractvalue { <4 x float>, <4 x float> } %9, 1
+ %11 = fmul fast <4 x float> %10, %8
+ %12 = fadd fast <4 x float> %11, %6
+ store <4 x float> %12, ptr %next.gep24, align 4
+ %index.next = add i64 %index, 4
+ %13 = icmp eq i64 %index.next, 1024
+ br i1 %13, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @vld2_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-IAENABLED-LABEL: vld2_multiuse_intrinsic:
+; CHECK-IAENABLED: .Lfunc_begin21:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB21_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; CHECK-IAENABLED-NEXT: fmul v2.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v2.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: str q2, [x1, x8]
+; CHECK-IAENABLED-NEXT: add x8, x8, #16
+; CHECK-IAENABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IAENABLED-NEXT: b.ne .LBB21_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2_multiuse_intrinsic:
+; CHECK-IADISABLED: .Lfunc_begin21:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB21_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0], #32
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmla v1.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: str q1, [x1, x8]
+; CHECK-IADISABLED-NEXT: add x8, x8, #16
+; CHECK-IADISABLED-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-IADISABLED-NEXT: b.ne .LBB21_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = shl i64 %index, 1
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %next.gep19 = getelementptr float, ptr %pDst, i64 %index
+ %wide.vec = load <8 x float>, ptr %next.gep, align 4
+ %1 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %wide.vec)
+ %2 = extractvalue { <4 x float>, <4 x float> } %1, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %1, 1
+ %4 = fmul fast <4 x float> %2, %2
+ %5 = fmul fast <4 x float> %3, %3
+ %6 = fadd fast <4 x float> %5, %4
+ store <4 x float> %6, ptr %next.gep19, align 4
+ %index.next = add i64 %index, 4
+ %7 = icmp eq i64 %index.next, 1024
+ br i1 %7, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @vld3_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-LABEL: vld3_multiuse_intrinsic:
+; CHECK: .Lfunc_begin22:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .LBB22_1: // %vector.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-NEXT: str q3, [x1, x8]
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-NEXT: b.ne .LBB22_1
+; CHECK-NEXT: // %bb.2: // %while.end
+; CHECK-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = mul i64 %index, 3
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %next.gep23 = getelementptr float, ptr %pDst, i64 %index
+ %wide.vec = load <12 x float>, ptr %next.gep, align 4
+ %1 = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %wide.vec)
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %1, 0
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %1, 1
+ %4 = fmul fast <4 x float> %2, %2
+ %5 = fmul fast <4 x float> %3, %3
+ %6 = fadd fast <4 x float> %5, %4
+ %7 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %1, 2
+ %8 = fmul fast <4 x float> %7, %7
+ %9 = fadd fast <4 x float> %6, %8
+ store <4 x float> %9, ptr %next.gep23, align 4
+ %index.next = add i64 %index, 4
+ %10 = icmp eq i64 %index.next, 1024
+ br i1 %10, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+
+define void @vld4_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
+; CHECK-IAENABLED-LABEL: vld4_multiuse_intrinsic:
+; CHECK-IAENABLED: .Lfunc_begin23:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov x8, xzr
+; CHECK-IAENABLED-NEXT: .LBB23_1: // %vector.body
+; CHECK-IAENABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IAENABLED-NEXT: add x9, x1, x8
+; CHECK-IAENABLED-NEXT: add x8, x8, #32
+; CHECK-IAENABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IAENABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IAENABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x9]
+; CHECK-IAENABLED-NEXT: b.ne .LBB23_1
+; CHECK-IAENABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4_multiuse_intrinsic:
+; CHECK-IADISABLED: .Lfunc_begin23:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: mov x8, xzr
+; CHECK-IADISABLED-NEXT: .LBB23_1: // %vector.body
+; CHECK-IADISABLED-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-IADISABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-IADISABLED-NEXT: add x9, x1, x8
+; CHECK-IADISABLED-NEXT: add x8, x8, #32
+; CHECK-IADISABLED-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v0.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v4.4s, v5.4s
+; CHECK-IADISABLED-NEXT: stp q1, q0, [x9]
+; CHECK-IADISABLED-NEXT: b.ne .LBB23_1
+; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %0 = shl i64 %index, 2
+ %next.gep = getelementptr float, ptr %pSrc, i64 %0
+ %1 = shl i64 %index, 1
+ %wide.vec = load <16 x float>, ptr %next.gep, align 4
+ %2 = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %wide.vec)
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %2, 0
+ %4 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %2, 1
+ %5 = fmul fast <4 x float> %3, %3
+ %6 = fmul fast <4 x float> %4, %4
+ %7 = fadd fast <4 x float> %6, %5
+ %8 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %2, 2
+ %9 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %2, 3
+ %10 = fmul fast <4 x float> %8, %8
+ %11 = fmul fast <4 x float> %9, %9
+ %12 = fadd fast <4 x float> %11, %10
+ %13 = getelementptr inbounds float, ptr %pDst, i64 %1
+ %interleaved.vec = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %7, <4 x float> %12)
+ store <8 x float> %interleaved.vec, ptr %13, align 4
+ %index.next = add i64 %index, 4
+ %14 = icmp eq i64 %index.next, 1024
+ br i1 %14, label %while.end, label %vector.body
+
+while.end: ; preds = %vector.body
+ ret void
+}
+
+define void @transpose_s16_8x8_simpler_intrinsic(ptr nocapture noundef %a) {
+; CHECK-LABEL: transpose_s16_8x8_simpler_intrinsic:
+; CHECK: .Lfunc_begin24:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: ldp q0, q1, [x0]
+; CHECK-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: trn1 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: str q2, [x0]
+; CHECK-NEXT: str q0, [x0, #64]
+; CHECK-NEXT: ret
+entry:
+ %0 = load <8 x i16>, ptr %a, align 16
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %shuffle.i = shufflevector <8 x i16> %0, <8 x i16> %1, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 2
+ %2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %arrayidx3 = getelementptr inbounds <8 x i16>, ptr %a, i64 3
+ %3 = load <8 x i16>, ptr %arrayidx3, align 16
+ %shuffle.i34 = shufflevector <8 x i16> %2, <8 x i16> %3, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx5 = getelementptr inbounds <8 x i16>, ptr %a, i64 4
+ %4 = load <8 x i16>, ptr %arrayidx5, align 16
+ %arrayidx6 = getelementptr inbounds <8 x i16>, ptr %a, i64 5
+ %5 = load <8 x i16>, ptr %arrayidx6, align 16
+ %shuffle.i35 = shufflevector <8 x i16> %4, <8 x i16> %5, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx8 = getelementptr inbounds <8 x i16>, ptr %a, i64 6
+ %6 = load <8 x i16>, ptr %arrayidx8, align 16
+ %arrayidx9 = getelementptr inbounds <8 x i16>, ptr %a, i64 7
+ %7 = load <8 x i16>, ptr %arrayidx9, align 16
+ %shuffle.i36 = shufflevector <8 x i16> %6, <8 x i16> %7, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %8 = bitcast <8 x i16> %shuffle.i to <4 x i32>
+ %9 = bitcast <8 x i16> %shuffle.i35 to <4 x i32>
+ %shuffle.i37 = shufflevector <4 x i32> %8, <4 x i32> %9, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %10 = bitcast <8 x i16> %shuffle.i34 to <4 x i32>
+ %11 = bitcast <8 x i16> %shuffle.i36 to <4 x i32>
+ %shuffle.i38 = shufflevector <4 x i32> %10, <4 x i32> %11, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %vzip = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %shuffle.i37, <4 x i32> %shuffle.i38)
+ %vzip.i = shufflevector <8 x i32> %vzip, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %vzip1.i = shufflevector <8 x i32> %vzip, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ store <4 x i32> %vzip.i, ptr %a, align 16
+ store <4 x i32> %vzip1.i, ptr %arrayidx5, align 16
+ ret void
+}
+
+define void @transpose_s16_8x8_simpler2_intrinsic(ptr nocapture noundef %a) {
+; CHECK-LABEL: transpose_s16_8x8_simpler2_intrinsic:
+; CHECK: .Lfunc_begin25:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0: // %entry
+; CHECK-NEXT: ldp q0, q1, [x0]
+; CHECK-NEXT: ldp q2, q3, [x0, #64]
+; CHECK-NEXT: ldp q4, q5, [x0, #96]
+; CHECK-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: zip1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-NEXT: ldp q3, q4, [x0, #32]
+; CHECK-NEXT: zip1 v3.8h, v3.8h, v4.8h
+; CHECK-NEXT: dup v1.4s, v1.s[0]
+; CHECK-NEXT: uzp1 v2.4s, v0.4s, v2.4s
+; CHECK-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: str q1, [x0]
+; CHECK-NEXT: str q0, [x0, #64]
+; CHECK-NEXT: ret
+entry:
+ %0 = load <8 x i16>, ptr %a, align 16
+ %arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
+ %1 = load <8 x i16>, ptr %arrayidx1, align 16
+ %shuffle.i = shufflevector <8 x i16> %0, <8 x i16> %1, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx2 = getelementptr inbounds <8 x i16>, ptr %a, i64 2
+ %2 = load <8 x i16>, ptr %arrayidx2, align 16
+ %arrayidx3 = getelementptr inbounds <8 x i16>, ptr %a, i64 3
+ %3 = load <8 x i16>, ptr %arrayidx3, align 16
+ %shuffle.i34 = shufflevector <8 x i16> %2, <8 x i16> %3, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx5 = getelementptr inbounds <8 x i16>, ptr %a, i64 4
+ %4 = load <8 x i16>, ptr %arrayidx5, align 16
+ %arrayidx6 = getelementptr inbounds <8 x i16>, ptr %a, i64 5
+ %5 = load <8 x i16>, ptr %arrayidx6, align 16
+ %shuffle.i35 = shufflevector <8 x i16> %4, <8 x i16> %5, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %arrayidx8 = getelementptr inbounds <8 x i16>, ptr %a, i64 6
+ %6 = load <8 x i16>, ptr %arrayidx8, align 16
+ %arrayidx9 = getelementptr inbounds <8 x i16>, ptr %a, i64 7
+ %7 = load <8 x i16>, ptr %arrayidx9, align 16
+ %shuffle.i36 = shufflevector <8 x i16> %6, <8 x i16> %7, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 undef, i32 undef>
+ %8 = bitcast <8 x i16> %shuffle.i to <4 x i32>
+ %9 = bitcast <8 x i16> %shuffle.i35 to <4 x i32>
+ %shuffle.i37 = shufflevector <4 x i32> %8, <4 x i32> %9, <4 x i32> <i32 1, i32 4, i32 2, i32 7>
+ %10 = bitcast <8 x i16> %shuffle.i34 to <4 x i32>
+ %11 = bitcast <8 x i16> %shuffle.i36 to <4 x i32>
+ %shuffle.i38 = shufflevector <4 x i32> %10, <4 x i32> %11, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
+ %vzip = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %shuffle.i37, <4 x i32> %shuffle.i38)
+ %vzip.i = shufflevector <8 x i32> %vzip, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %vzip1.i = shufflevector <8 x i32> %vzip, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ store <4 x i32> %vzip.i, ptr %a, align 16
+ store <4 x i32> %vzip1.i, ptr %arrayidx5, align 16
+ ret void
+}
+
+define void @transpose_s16_8x8_intrinsic(ptr nocapture noundef %0, ptr nocapture noundef %1, ptr nocapture noundef %2, ptr nocapture noundef %3, ptr nocapture noundef %4, ptr nocapture noundef %5, ptr nocapture noundef %6, ptr nocapture noundef %7) {
+; CHECK-LABEL: transpose_s16_8x8_intrinsic:
+; CHECK: .Lfunc_begin26:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ldr q0, [x2]
+; CHECK-NEXT: ldr q1, [x0]
+; CHECK-NEXT: ldr q2, [x1]
+; CHECK-NEXT: ldr q3, [x3]
+; CHECK-NEXT: ldr q4, [x4]
+; CHECK-NEXT: ldr q5, [x5]
+; CHECK-NEXT: trn1 v16.8h, v1.8h, v2.8h
+; CHECK-NEXT: trn2 v1.8h, v1.8h, v2.8h
+; CHECK-NEXT: ldr q6, [x6]
+; CHECK-NEXT: ldr q7, [x7]
+; CHECK-NEXT: trn1 v2.8h, v0.8h, v3.8h
+; CHECK-NEXT: trn1 v17.8h, v4.8h, v5.8h
+; CHECK-NEXT: trn2 v0.8h, v0.8h, v3.8h
+; CHECK-NEXT: trn2 v3.8h, v4.8h, v5.8h
+; CHECK-NEXT: trn1 v18.8h, v6.8h, v7.8h
+; CHECK-NEXT: trn2 v4.8h, v6.8h, v7.8h
+; CHECK-NEXT: trn1 v5.4s, v16.4s, v17.4s
+; CHECK-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-NEXT: trn1 v7.4s, v1.4s, v3.4s
+; CHECK-NEXT: trn2 v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: trn1 v6.4s, v2.4s, v18.4s
+; CHECK-NEXT: trn1 v19.4s, v0.4s, v4.4s
+; CHECK-NEXT: trn2 v2.4s, v2.4s, v18.4s
+; CHECK-NEXT: trn2 v0.4s, v0.4s, v4.4s
+; CHECK-NEXT: zip1 v3.4s, v5.4s, v6.4s
+; CHECK-NEXT: zip1 v4.4s, v7.4s, v19.4s
+; CHECK-NEXT: zip1 v17.4s, v16.4s, v2.4s
+; CHECK-NEXT: zip1 v18.4s, v1.4s, v0.4s
+; CHECK-NEXT: zip2 v5.4s, v5.4s, v6.4s
+; CHECK-NEXT: zip2 v2.4s, v16.4s, v2.4s
+; CHECK-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: str q3, [x0]
+; CHECK-NEXT: zip2 v3.4s, v7.4s, v19.4s
+; CHECK-NEXT: str q4, [x1]
+; CHECK-NEXT: str q17, [x2]
+; CHECK-NEXT: str q18, [x3]
+; CHECK-NEXT: str q5, [x4]
+; CHECK-NEXT: str q3, [x5]
+; CHECK-NEXT: str q2, [x6]
+; CHECK-NEXT: str q0, [x7]
+; CHECK-NEXT: ret
+ %9 = load <8 x i16>, ptr %0, align 16
+ %10 = load <8 x i16>, ptr %1, align 16
+ %11 = shufflevector <8 x i16> %9, <8 x i16> %10, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+ %12 = shufflevector <8 x i16> %9, <8 x i16> %10, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+ %13 = load <8 x i16>, ptr %2, align 16
+ %14 = load <8 x i16>, ptr %3, align 16
+ %15 = shufflevector <8 x i16> %13, <8 x i16> %14, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+ %16 = shufflevector <8 x i16> %13, <8 x i16> %14, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+ %17 = load <8 x i16>, ptr %4, align 16
+ %18 = load <8 x i16>, ptr %5, align 16
+ %19 = shufflevector <8 x i16> %17, <8 x i16> %18, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+ %20 = shufflevector <8 x i16> %17, <8 x i16> %18, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+ %21 = load <8 x i16>, ptr %6, align 16
+ %22 = load <8 x i16>, ptr %7, align 16
+ %23 = shufflevector <8 x i16> %21, <8 x i16> %22, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+ %24 = shufflevector <8 x i16> %21, <8 x i16> %22, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+ %25 = bitcast <8 x i16> %11 to <4 x i32>
+ %26 = bitcast <8 x i16> %19 to <4 x i32>
+ %27 = shufflevector <4 x i32> %25, <4 x i32> %26, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %28 = shufflevector <4 x i32> %25, <4 x i32> %26, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+ %29 = bitcast <8 x i16> %12 to <4 x i32>
+ %30 = bitcast <8 x i16> %20 to <4 x i32>
+ %31 = shufflevector <4 x i32> %29, <4 x i32> %30, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %32 = shufflevector <4 x i32> %29, <4 x i32> %30, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+ %33 = bitcast <8 x i16> %15 to <4 x i32>
+ %34 = bitcast <8 x i16> %23 to <4 x i32>
+ %35 = shufflevector <4 x i32> %33, <4 x i32> %34, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %36 = shufflevector <4 x i32> %33, <4 x i32> %34, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+ %37 = bitcast <8 x i16> %16 to <4 x i32>
+ %38 = bitcast <8 x i16> %24 to <4 x i32>
+ %39 = shufflevector <4 x i32> %37, <4 x i32> %38, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %40 = shufflevector <4 x i32> %37, <4 x i32> %38, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+ %interleave.41 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %27, <4 x i32> %35)
+ %41 = shufflevector <8 x i32> %interleave.41, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %42 = shufflevector <8 x i32> %interleave.41, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %interleave.43 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %31, <4 x i32> %39)
+ %43 = shufflevector <8 x i32> %interleave.43, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %44 = shufflevector <8 x i32> %interleave.43, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %interleave.45 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %28, <4 x i32> %36)
+ %45 = shufflevector <8 x i32> %interleave.45, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %46 = shufflevector <8 x i32> %interleave.45, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %interleave.47 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %32, <4 x i32> %40)
+ %47 = shufflevector <8 x i32> %interleave.47, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %48 = shufflevector <8 x i32> %interleave.47, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ store <4 x i32> %41, ptr %0, align 16
+ store <4 x i32> %43, ptr %1, align 16
+ store <4 x i32> %45, ptr %2, align 16
+ store <4 x i32> %47, ptr %3, align 16
+ store <4 x i32> %42, ptr %4, align 16
+ store <4 x i32> %44, ptr %5, align 16
+ store <4 x i32> %46, ptr %6, align 16
+ store <4 x i32> %48, ptr %7, align 16
+ ret void
+}
+
+define void @store_factor2_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-IAENABLED-LABEL: store_factor2_intrinsic:
+; CHECK-IAENABLED: .Lfunc_begin27:
+; CHECK-IAENABLED-NEXT: .cfi_startproc
+; CHECK-IAENABLED-NEXT: // %bb.0:
+; CHECK-IAENABLED-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-IAENABLED-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor2_intrinsic:
+; CHECK-IADISABLED: .Lfunc_begin27:
+; CHECK-IADISABLED-NEXT: .cfi_startproc
+; CHECK-IADISABLED-NEXT: // %bb.0:
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: ext v0.16b, v2.16b, v0.16b, #8
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ret
+ %v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2) {
+; CHECK-LABEL: store_factor3_intrinsic:
+; CHECK: .Lfunc_begin28:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ext v3.16b, v0.16b, v1.16b, #12
+; CHECK-NEXT: ext v6.16b, v1.16b, v2.16b, #12
+; CHECK-NEXT: zip2 v3.4s, v0.4s, v3.4s
+; CHECK-NEXT: mov v3.s[0], v0.s[0]
+; CHECK-NEXT: ext v0.16b, v2.16b, v0.16b, #12
+; CHECK-NEXT: zip2 v4.4s, v1.4s, v6.4s
+; CHECK-NEXT: mov v4.s[0], v1.s[0]
+; CHECK-NEXT: zip2 v5.4s, v2.4s, v0.4s
+; CHECK-NEXT: mov v5.s[0], v2.s[0]
+; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-NEXT: ret
+ %v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
+ %v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
+ %v2 = shufflevector <4 x i32> %a2, <4 x i32> %a0, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
+ %interleaved.vec = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x i32> %a3) {
+; CHECK-LABEL: store_factor4_intrinsic:
+; CHECK: .Lfunc_begin29:
+; CHECK-NEXT: .cfi_startproc
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: trn1 v4.4s, v0.4s, v1.4s
+; CHECK-NEXT: trn1 v5.4s, v1.4s, v2.4s
+; CHECK-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-NEXT: trn1 v7.4s, v3.4s, v0.4s
+; CHECK-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-NEXT: ret
+ %v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %v2 = shufflevector <4 x i32> %a2, <4 x i32> %a3, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %v3 = shufflevector <4 x i32> %a3, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %interleaved.vec = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
+ store <16 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!6, !7, !8, !9, !10, !11}
diff --git a/llvm/test/CodeGen/AArch64/zext-shuffle.ll b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
index cf917341f18b0..0b50735e0a367 100644
--- a/llvm/test/CodeGen/AArch64/zext-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-none-eabi -o - %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false -mtriple=aarch64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
define <2 x i64> @v2i64_02(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: v2i64_02:
@@ -371,18 +372,41 @@ define <8 x i64> @zext_add(<32 x i16> %l) {
}
define <8 x i64> @zext_load_add(ptr %p) {
-; CHECK-LABEL: zext_load_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
-; CHECK-NEXT: uaddl v4.4s, v0.4h, v1.4h
-; CHECK-NEXT: uaddl v5.4s, v2.4h, v3.4h
-; CHECK-NEXT: uaddl2 v6.4s, v0.8h, v1.8h
-; CHECK-NEXT: uaddl2 v2.4s, v2.8h, v3.8h
-; CHECK-NEXT: uaddl v0.2d, v4.2s, v5.2s
-; CHECK-NEXT: uaddl2 v1.2d, v4.4s, v5.4s
-; CHECK-NEXT: uaddl2 v3.2d, v6.4s, v2.4s
-; CHECK-NEXT: uaddl v2.2d, v6.2s, v2.2s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: zext_load_add:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-IAENABLED-NEXT: uaddl v4.4s, v0.4h, v1.4h
+; CHECK-IAENABLED-NEXT: uaddl v5.4s, v2.4h, v3.4h
+; CHECK-IAENABLED-NEXT: uaddl2 v6.4s, v0.8h, v1.8h
+; CHECK-IAENABLED-NEXT: uaddl2 v2.4s, v2.8h, v3.8h
+; CHECK-IAENABLED-NEXT: uaddl v0.2d, v4.2s, v5.2s
+; CHECK-IAENABLED-NEXT: uaddl2 v1.2d, v4.4s, v5.4s
+; CHECK-IAENABLED-NEXT: uaddl2 v3.2d, v6.4s, v2.4s
+; CHECK-IAENABLED-NEXT: uaddl v2.2d, v6.2s, v2.2s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: zext_load_add:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q2, q1, [x0]
+; CHECK-IADISABLED-NEXT: movi v0.2d, #0x00ffff0000ffff
+; CHECK-IADISABLED-NEXT: ldp q4, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v4.4s, v3.4s
+; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-IADISABLED-NEXT: and v4.16b, v5.16b, v0.16b
+; CHECK-IADISABLED-NEXT: and v6.16b, v1.16b, v0.16b
+; CHECK-IADISABLED-NEXT: and v7.16b, v2.16b, v0.16b
+; CHECK-IADISABLED-NEXT: and v16.16b, v3.16b, v0.16b
+; CHECK-IADISABLED-NEXT: usra v4.4s, v5.4s, #16
+; CHECK-IADISABLED-NEXT: usra v6.4s, v1.4s, #16
+; CHECK-IADISABLED-NEXT: usra v7.4s, v2.4s, #16
+; CHECK-IADISABLED-NEXT: usra v16.4s, v3.4s, #16
+; CHECK-IADISABLED-NEXT: uaddl v0.2d, v4.2s, v6.2s
+; CHECK-IADISABLED-NEXT: uaddl2 v1.2d, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT: uaddl2 v3.2d, v7.4s, v16.4s
+; CHECK-IADISABLED-NEXT: uaddl v2.2d, v7.2s, v16.2s
+; CHECK-IADISABLED-NEXT: ret
%l = load <32 x i16>, ptr %p
%s1 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%z1 = zext <8 x i16> %s1 to <8 x i64>
@@ -739,5 +763,35 @@ entry:
ret <4 x i32> %1
}
+define <8 x i64> @zext_load_add_intrinsic(ptr %p) {
+; CHECK-LABEL: zext_load_add_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-NEXT: uaddl v4.4s, v0.4h, v1.4h
+; CHECK-NEXT: uaddl v5.4s, v2.4h, v3.4h
+; CHECK-NEXT: uaddl2 v6.4s, v0.8h, v1.8h
+; CHECK-NEXT: uaddl2 v2.4s, v2.8h, v3.8h
+; CHECK-NEXT: uaddl v0.2d, v4.2s, v5.2s
+; CHECK-NEXT: uaddl2 v1.2d, v4.4s, v5.4s
+; CHECK-NEXT: uaddl2 v3.2d, v6.4s, v2.4s
+; CHECK-NEXT: uaddl v2.2d, v6.2s, v2.2s
+; CHECK-NEXT: ret
+ %l = load <32 x i16>, ptr %p
+ %deint = call {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} @llvm.vector.deinterleave4(<32 x i16>%l)
+ %s1 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 0
+ %z1 = zext <8 x i16> %s1 to <8 x i64>
+ %s2 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 1
+ %z2 = zext <8 x i16> %s2 to <8 x i64>
+ %s3 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 2
+ %z3 = zext <8 x i16> %s3 to <8 x i64>
+ %s4 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 3
+ %z4 = zext <8 x i16> %s4 to <8 x i64>
+ %a = add <8 x i64> %z1, %z2
+ %b = add <8 x i64> %z3, %z4
+ %c = add <8 x i64> %a, %b
+ ret <8 x i64> %c
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
attributes #1 = { vscale_range(2,2) "target-features"="+sve" }
+
More information about the llvm-commits
mailing list