[llvm] [AArch64] Move interleaved access testcase to codegen (PR #214133)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 01:50:39 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/214133
>From b34a7238548ed3fe6b12f9b598289ab9569a877b Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 29 Jul 2026 09:30:46 +0000
Subject: [PATCH 1/5] [AArch64] Move test to codegen/aarch64
---
.../AArch64/binopshuffles-inseltpoison.ll | 0
.../InterleavedAccess => CodeGen}/AArch64/binopshuffles.ll | 0
.../AArch64/fixed-deinterleave-intrinsics.ll | 0
.../AArch64/interleaved-accesses-extract-user-inseltpoison.ll | 0
.../AArch64/interleaved-accesses-extract-user.ll | 0
.../InterleavedAccess => CodeGen}/AArch64/interleaved-accesses.ll | 0
.../AArch64/sve-interleaved-accesses.ll | 0
7 files changed, 0 insertions(+), 0 deletions(-)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/binopshuffles-inseltpoison.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/binopshuffles.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/fixed-deinterleave-intrinsics.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/interleaved-accesses-extract-user-inseltpoison.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/interleaved-accesses-extract-user.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/interleaved-accesses.ll (100%)
rename llvm/test/{Transforms/InterleavedAccess => CodeGen}/AArch64/sve-interleaved-accesses.ll (100%)
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll
rename to llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll
rename to llvm/test/CodeGen/AArch64/binopshuffles.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll
rename to llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
rename to llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll
rename to llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
rename to llvm/test/CodeGen/AArch64/interleaved-accesses.ll
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
similarity index 100%
rename from llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
rename to llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
>From a12c4c0b12aa504691a6a1c18a456e25c2d6b040 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 29 Jul 2026 10:09:26 +0000
Subject: [PATCH 2/5] [AArch64] update test with llc
---
.../AArch64/binopshuffles-inseltpoison.ll | 111 +-
llvm/test/CodeGen/AArch64/binopshuffles.ll | 187 +--
.../AArch64/fixed-deinterleave-intrinsics.ll | 487 +++----
...aved-accesses-extract-user-inseltpoison.ll | 61 +-
.../interleaved-accesses-extract-user.ll | 61 +-
.../CodeGen/AArch64/interleaved-accesses.ll | 1228 +++++++++++------
.../AArch64/sve-interleaved-accesses.ll | 614 ++++-----
7 files changed, 1443 insertions(+), 1306 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
index 22df002dd62cd..7f7e6ce967b01 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
@@ -1,23 +1,16 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -interleaved-access -S | FileCheck %s
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define <4 x float> @vld2(ptr %pSrc) {
-; CHECK-LABEL: @vld2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L26:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[L43:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[L6:%.*]] = fadd fast <4 x float> [[L43]], [[L26]]
-; CHECK-NEXT: ret <4 x float> [[L6]]
-;
+; CHECK-LABEL: vld2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -29,22 +22,13 @@ entry:
}
define <4 x float> @vld3(ptr %pSrc) {
-; CHECK-LABEL: @vld3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld3.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L29:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[L46:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[L6:%.*]] = fadd fast <4 x float> [[L46]], [[L29]]
-; CHECK-NEXT: [[L73:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[L9:%.*]] = fadd fast <4 x float> [[L6]], [[L73]]
-; CHECK-NEXT: ret <4 x float> [[L9]]
-;
+; CHECK-LABEL: vld3:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
@@ -59,25 +43,12 @@ entry:
}
define <4 x float> @vld4(ptr %pSrc) {
-; CHECK-LABEL: @vld4(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld4.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L312:%.*]] = fmul fast <4 x float> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[L59:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[L7:%.*]] = fadd fast <4 x float> [[L59]], [[L312]]
-; CHECK-NEXT: [[L86:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[L103:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[L12:%.*]] = fadd fast <4 x float> [[L103]], [[L86]]
-; CHECK-NEXT: ret <4 x float> [[L12]]
-;
+; CHECK-LABEL: vld4:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
@@ -94,19 +65,13 @@ entry:
}
define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: @twosrc(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC1:%.*]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[LDN7:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC2:%.*]])
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN7]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN7]], 1
-; CHECK-NEXT: [[L46:%.*]] = fmul fast <4 x float> [[TMP4]], [[TMP2]]
-; CHECK-NEXT: [[L63:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP1]]
-; CHECK-NEXT: [[L8:%.*]] = fadd fast <4 x float> [[L63]], [[L46]]
-; CHECK-NEXT: ret <4 x float> [[L8]]
-;
+; CHECK-LABEL: twosrc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -119,19 +84,13 @@ entry:
}
define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: @twosrc2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC1:%.*]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[LDN4:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC2:%.*]])
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN4]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN4]], 1
-; CHECK-NEXT: [[L43:%.*]] = fmul fast <4 x float> [[TMP4]], [[TMP2]]
-; CHECK-NEXT: [[L6:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP1]]
-; CHECK-NEXT: [[L8:%.*]] = fadd fast <4 x float> [[L6]], [[L43]]
-; CHECK-NEXT: ret <4 x float> [[L8]]
-;
+; CHECK-LABEL: twosrc2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 399fa5298b7cc..519fed793603e 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -1,23 +1,16 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -interleaved-access -S | FileCheck %s
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define <4 x float> @vld2(ptr %pSrc) {
-; CHECK-LABEL: @vld2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L26:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[L43:%.*]] = fmul fast <4 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[L6:%.*]] = fadd fast <4 x float> [[L43]], [[L26]]
-; CHECK-NEXT: ret <4 x float> [[L6]]
-;
+; CHECK-LABEL: vld2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -29,22 +22,13 @@ entry:
}
define <4 x float> @vld3(ptr %pSrc) {
-; CHECK-LABEL: @vld3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld3.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L29:%.*]] = fmul fast <4 x float> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[L46:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[L6:%.*]] = fadd fast <4 x float> [[L46]], [[L29]]
-; CHECK-NEXT: [[L73:%.*]] = fmul fast <4 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[L9:%.*]] = fadd fast <4 x float> [[L6]], [[L73]]
-; CHECK-NEXT: ret <4 x float> [[L9]]
-;
+; CHECK-LABEL: vld3:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
@@ -59,25 +43,12 @@ entry:
}
define <4 x float> @vld4(ptr %pSrc) {
-; CHECK-LABEL: @vld4(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld4.v4f32.p0(ptr [[PSRC:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 3
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[L312:%.*]] = fmul fast <4 x float> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: [[L59:%.*]] = fmul fast <4 x float> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[L7:%.*]] = fadd fast <4 x float> [[L59]], [[L312]]
-; CHECK-NEXT: [[L86:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[L103:%.*]] = fmul fast <4 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[L12:%.*]] = fadd fast <4 x float> [[L103]], [[L86]]
-; CHECK-NEXT: ret <4 x float> [[L12]]
-;
+; CHECK-LABEL: vld4:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
@@ -94,19 +65,13 @@ entry:
}
define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: @twosrc(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC1:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[LDN7:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC2:%.*]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN7]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN7]], 1
-; CHECK-NEXT: [[L46:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT: [[L63:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP0]]
-; CHECK-NEXT: [[L8:%.*]] = fadd fast <4 x float> [[L63]], [[L46]]
-; CHECK-NEXT: ret <4 x float> [[L8]]
-;
+; CHECK-LABEL: twosrc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -119,19 +84,13 @@ entry:
}
define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: @twosrc2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC1:%.*]])
-; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[LDN4:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PSRC2:%.*]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN4]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN4]], 1
-; CHECK-NEXT: [[L43:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT: [[L6:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP0]]
-; CHECK-NEXT: [[L8:%.*]] = fadd fast <4 x float> [[L6]], [[L43]]
-; CHECK-NEXT: ret <4 x float> [[L8]]
-;
+; CHECK-LABEL: twosrc2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -145,15 +104,12 @@ entry:
}
define void @noncanonical(ptr %p0, ptr %p1, ptr %p2) {
-; CHECK-LABEL: @noncanonical(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = load <8 x i8>, ptr [[P0:%.*]], align 8
-; CHECK-NEXT: [[V1:%.*]] = add <8 x i8> [[V0]], <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
-; CHECK-NEXT: [[V2:%.*]] = load <8 x i8>, ptr [[P1:%.*]], align 8
-; CHECK-NEXT: [[SHUFFLED:%.*]] = shufflevector <8 x i8> [[V2]], <8 x i8> [[V1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: store <4 x i8> [[SHUFFLED]], ptr [[P2:%.*]], align 4
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: noncanonical:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr d0, [x1]
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: str s0, [x2]
+; CHECK-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = add <8 x i8> %v0, <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
@@ -164,15 +120,9 @@ entry:
}
define void @noncanonical2(ptr %p0, ptr %p1, ptr %p2) {
-; CHECK-LABEL: @noncanonical2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = load <8 x i8>, ptr [[P0:%.*]], align 8
-; CHECK-NEXT: [[V1:%.*]] = load <8 x i8>, ptr [[P1:%.*]], align 8
-; CHECK-NEXT: [[V2:%.*]] = add <8 x i8> [[V0]], [[V1]]
-; CHECK-NEXT: [[SHUFFLED:%.*]] = shufflevector <8 x i8> undef, <8 x i8> [[V2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: store <4 x i8> [[SHUFFLED]], ptr [[P2:%.*]], align 4
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: noncanonical2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = load <8 x i8>, ptr %p1
@@ -183,16 +133,13 @@ entry:
}
define <4 x float> @noncanonical3(ptr %pSrc) {
-; CHECK-LABEL: @noncanonical3(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[PSRC:%.*]], align 4
-; CHECK-NEXT: [[L2:%.*]] = fmul fast <8 x float> [[WIDE_VEC]], [[WIDE_VEC]]
-; CHECK-NEXT: [[L3:%.*]] = shufflevector <8 x float> undef, <8 x float> [[L2]], <4 x i32> <i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[L4:%.*]] = fmul fast <8 x float> [[WIDE_VEC]], [[WIDE_VEC]]
-; CHECK-NEXT: [[L5:%.*]] = shufflevector <8 x float> [[L4]], <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT: [[L6:%.*]] = fadd fast <4 x float> [[L5]], [[L3]]
-; CHECK-NEXT: ret <4 x float> [[L6]]
-;
+; CHECK-LABEL: noncanonical3:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q0, q1, [x0]
+; CHECK-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -204,15 +151,16 @@ entry:
}
define void @noncanonical_extmask(ptr %p0, ptr %p1, ptr %p2) {
-; CHECK-LABEL: @noncanonical_extmask(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = load <8 x i8>, ptr [[P0:%.*]], align 8
-; CHECK-NEXT: [[V1:%.*]] = add <8 x i8> [[V0]], <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
-; CHECK-NEXT: [[V2:%.*]] = load <8 x i8>, ptr [[P1:%.*]], align 8
-; CHECK-NEXT: [[SHUFFLED:%.*]] = shufflevector <8 x i8> [[V2]], <8 x i8> [[V1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: store <8 x i8> [[SHUFFLED]], ptr [[P2:%.*]], align 8
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: noncanonical_extmask:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: adrp x8, .LCPI8_0
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI8_0]
+; CHECK-NEXT: add v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: str d0, [x2]
+; CHECK-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = add <8 x i8> %v0, <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
@@ -223,16 +171,9 @@ entry:
}
define void @skip_optimizing_dead_binop(ptr %p0, ptr %p1) {
-; CHECK-LABEL: @skip_optimizing_dead_binop(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[V0:%.*]] = load <8 x double>, ptr [[P0:%.*]]
-; CHECK-NEXT: [[SHUFFLED_1:%.*]] = shufflevector <8 x double> [[V0]], <8 x double> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT: [[SHUFFLED_2:%.*]] = shufflevector <8 x double> [[V0]], <8 x double> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT: [[SHUFFLED_3:%.*]] = shufflevector <8 x double> [[V0]], <8 x double> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT: [[SHUFFLED_4:%.*]] = shufflevector <8 x double> [[V0]], <8 x double> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT: [[DEAD_BINOP:%.*]] = fadd <8 x double> [[V0]], [[V0]]
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: skip_optimizing_dead_binop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%v0 = load <8 x double>, ptr %p0
%shuffled_1 = shufflevector <8 x double> %v0, <8 x double> undef, <2 x i32> <i32 0, i32 4>
diff --git a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
index 6c81d9a4d2ed6..3f5d44af1d1e0 100644
--- a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
@@ -1,27 +1,17 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON
-; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible -S | FileCheck %s --check-prefix=SVE-FIXED
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON
-; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible -S | FileCheck %s --check-prefix=SVE-FIXED
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s | FileCheck %s --check-prefix=NEON
+; RUN: llc < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefix=SVE-FIXED
target triple = "aarch64-linux-gnu"
define void @deinterleave_i8_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_i8_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2.v16i8.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_i8_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <32 x i8>, ptr [[PTR]], align 1
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_i8_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_i8_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <32 x i8>, ptr %ptr, align 1
%deinterleave = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %load)
%extract1 = extractvalue { <16 x i8>, <16 x i8> } %deinterleave, 0
@@ -30,21 +20,13 @@ define void @deinterleave_i8_factor2(ptr %ptr) {
}
define void @deinterleave_i16_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_i16_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_i16_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <16 x i16>, ptr [[PTR]], align 2
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_i16_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_i16_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <16 x i16>, ptr %ptr, align 2
%deinterleave = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %load)
%extract1 = extractvalue { <8 x i16>, <8 x i16> } %deinterleave, 0
@@ -53,21 +35,13 @@ define void @deinterleave_i16_factor2(ptr %ptr) {
}
define void @deinterleave_8xi32_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_8xi32_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_8xi32_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <8 x i32>, ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_8xi32_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_8xi32_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 4
%deinterleave = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %load)
%extract1 = extractvalue { <4 x i32>, <4 x i32> } %deinterleave, 0
@@ -76,21 +50,13 @@ define void @deinterleave_8xi32_factor2(ptr %ptr) {
}
define void @deinterleave_i64_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_i64_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_i64_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <4 x i64>, ptr [[PTR]], align 8
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_i64_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_i64_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <4 x i64>, ptr %ptr, align 8
%deinterleave = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %load)
%extract1 = extractvalue { <2 x i64>, <2 x i64> } %deinterleave, 0
@@ -99,21 +65,13 @@ define void @deinterleave_i64_factor2(ptr %ptr) {
}
define void @deinterleave_float_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_float_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_float_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <8 x float>, ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_float_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_float_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <8 x float>, ptr %ptr, align 4
%deinterleave = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %load)
%extract1 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
@@ -122,21 +80,13 @@ define void @deinterleave_float_factor2(ptr %ptr) {
}
define void @deinterleave_double_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_double_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld2.v2f64.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_double_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <4 x double>, ptr [[PTR]], align 8
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_double_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_double_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <4 x double>, ptr %ptr, align 8
%deinterleave = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %load)
%extract1 = extractvalue { <2 x double>, <2 x double> } %deinterleave, 0
@@ -145,21 +95,13 @@ define void @deinterleave_double_factor2(ptr %ptr) {
}
define void @deinterleave_ptr_factor2(ptr %ptr) {
-; NEON-LABEL: define void @deinterleave_ptr_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x ptr>, <2 x ptr> } @llvm.aarch64.neon.ld2.v2p0.p0(ptr [[PTR]])
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[LDN]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[LDN]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_ptr_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <4 x ptr>, ptr [[PTR]], align 8
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_ptr_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_ptr_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <4 x ptr>, ptr %ptr, align 8
%deinterleave = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> %load)
%extract1 = extractvalue { <2 x ptr>, <2 x ptr> } %deinterleave, 0
@@ -168,153 +110,216 @@ define void @deinterleave_ptr_factor2(ptr %ptr) {
}
define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
-; NEON-LABEL: define void @interleave_i8_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <16 x i8> [[L:%.*]], <16 x i8> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[L]], <16 x i8> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_i8_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <16 x i8> [[L:%.*]], <16 x i8> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> [[L]], <16 x i8> [[R]])
-; SVE-FIXED-NEXT: store <32 x i8> [[INTERLEAVE]], ptr [[PTR]], align 1
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_i8_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i8_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
+; SVE-FIXED-NEXT: mov z3.b, z0.b[15]
+; SVE-FIXED-NEXT: mov z4.b, z1.b[14]
+; SVE-FIXED-NEXT: mov z5.b, z0.b[14]
+; SVE-FIXED-NEXT: mov z6.b, z1.b[13]
+; SVE-FIXED-NEXT: mov z7.b, z0.b[13]
+; SVE-FIXED-NEXT: mov z16.b, z1.b[12]
+; SVE-FIXED-NEXT: mov z17.b, z0.b[12]
+; SVE-FIXED-NEXT: mov z18.b, z1.b[11]
+; SVE-FIXED-NEXT: mov z19.b, z0.b[11]
+; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
+; SVE-FIXED-NEXT: mov z21.b, z0.b[10]
+; SVE-FIXED-NEXT: mov z22.b, z1.b[9]
+; SVE-FIXED-NEXT: mov z23.b, z0.b[9]
+; SVE-FIXED-NEXT: mov z24.b, z1.b[8]
+; SVE-FIXED-NEXT: mov z25.b, z0.b[8]
+; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
+; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
+; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
+; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
+; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
+; SVE-FIXED-NEXT: zip1 z7.b, z21.b, z20.b
+; SVE-FIXED-NEXT: zip1 z16.b, z23.b, z22.b
+; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z1.b
+; SVE-FIXED-NEXT: zip1 z17.b, z25.b, z24.b
+; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
+; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %l, <16 x i8> %r)
store <32 x i8> %interleave, ptr %ptr, align 1
ret void
}
define void @interleave_i16_factor2(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
-; NEON-LABEL: define void @interleave_i16_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <8 x i16> [[L:%.*]], <8 x i16> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v8i16.p0(<8 x i16> [[L]], <8 x i16> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_i16_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <8 x i16> [[L:%.*]], <8 x i16> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> [[L]], <8 x i16> [[R]])
-; SVE-FIXED-NEXT: store <16 x i16> [[INTERLEAVE]], ptr [[PTR]], align 2
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_i16_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i16_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.h, z1.h[7]
+; SVE-FIXED-NEXT: mov z3.h, z0.h[7]
+; SVE-FIXED-NEXT: mov z4.h, z1.h[6]
+; SVE-FIXED-NEXT: mov z5.h, z0.h[6]
+; SVE-FIXED-NEXT: mov z6.h, z1.h[5]
+; SVE-FIXED-NEXT: mov z7.h, z0.h[5]
+; SVE-FIXED-NEXT: mov z16.h, z1.h[4]
+; SVE-FIXED-NEXT: mov z17.h, z0.h[4]
+; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z1.h
+; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
+; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %l, <8 x i16> %r)
store <16 x i16> %interleave, ptr %ptr, align 2
ret void
}
define void @interleave_i32_factor2(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
-; NEON-LABEL: define void @interleave_i32_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <4 x i32> [[L:%.*]], <4 x i32> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[L]], <4 x i32> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_i32_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <4 x i32> [[L:%.*]], <4 x i32> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> [[L]], <4 x i32> [[R]])
-; SVE-FIXED-NEXT: store <8 x i32> [[INTERLEAVE]], ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_i32_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i32_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %l, <4 x i32> %r)
store <8 x i32> %interleave, ptr %ptr, align 4
ret void
}
define void @interleave_i64_factor2(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
-; NEON-LABEL: define void @interleave_i64_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <2 x i64> [[L:%.*]], <2 x i64> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[L]], <2 x i64> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_i64_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <2 x i64> [[L:%.*]], <2 x i64> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> [[L]], <2 x i64> [[R]])
-; SVE-FIXED-NEXT: store <4 x i64> [[INTERLEAVE]], ptr [[PTR]], align 8
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_i64_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i64_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %l, <2 x i64> %r)
store <4 x i64> %interleave, ptr %ptr, align 8
ret void
}
define void @interleave_float_factor2(ptr %ptr, <4 x float> %l, <4 x float> %r) {
-; NEON-LABEL: define void @interleave_float_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <4 x float> [[L:%.*]], <4 x float> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v4f32.p0(<4 x float> [[L]], <4 x float> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_float_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <4 x float> [[L:%.*]], <4 x float> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> [[L]], <4 x float> [[R]])
-; SVE-FIXED-NEXT: store <8 x float> [[INTERLEAVE]], ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_float_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_float_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %l, <4 x float> %r)
store <8 x float> %interleave, ptr %ptr, align 4
ret void
}
define void @interleave_double_factor2(ptr %ptr, <2 x double> %l, <2 x double> %r) {
-; NEON-LABEL: define void @interleave_double_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <2 x double> [[L:%.*]], <2 x double> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2f64.p0(<2 x double> [[L]], <2 x double> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_double_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <2 x double> [[L:%.*]], <2 x double> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> [[L]], <2 x double> [[R]])
-; SVE-FIXED-NEXT: store <4 x double> [[INTERLEAVE]], ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_double_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_double_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %l, <2 x double> %r)
store <4 x double> %interleave, ptr %ptr, align 4
ret void
}
define void @interleave_ptr_factor2(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
-; NEON-LABEL: define void @interleave_ptr_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <2 x ptr> [[L:%.*]], <2 x ptr> [[R:%.*]]) {
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2p0.p0(<2 x ptr> [[L]], <2 x ptr> [[R]], ptr [[PTR]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_ptr_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <2 x ptr> [[L:%.*]], <2 x ptr> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> [[L]], <2 x ptr> [[R]])
-; SVE-FIXED-NEXT: store <4 x ptr> [[INTERLEAVE]], ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_ptr_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_ptr_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %l, <2 x ptr> %r)
store <4 x ptr> %interleave, ptr %ptr, align 4
ret void
}
define void @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
-; NEON-LABEL: define void @deinterleave_wide_i16_factor2
-; NEON-SAME: (ptr [[PTR:%.*]]) {
-; NEON-NEXT: [[TMP1:%.*]] = getelementptr <8 x i16>, ptr [[PTR]], i64 0
-; NEON-NEXT: [[LDN:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr [[TMP1]])
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 0
-; NEON-NEXT: [[TMP3:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> poison, <8 x i16> [[TMP2]], i64 0)
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 1
-; NEON-NEXT: [[TMP5:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> poison, <8 x i16> [[TMP4]], i64 0)
-; NEON-NEXT: [[TMP6:%.*]] = getelementptr <8 x i16>, ptr [[PTR]], i64 2
-; NEON-NEXT: [[LDN1:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr [[TMP6]])
-; NEON-NEXT: [[TMP7:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN1]], 0
-; NEON-NEXT: [[TMP8:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> [[TMP3]], <8 x i16> [[TMP7]], i64 8)
-; NEON-NEXT: [[TMP9:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN1]], 1
-; NEON-NEXT: [[TMP10:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> [[TMP5]], <8 x i16> [[TMP9]], i64 8)
-; NEON-NEXT: [[TMP11:%.*]] = insertvalue { <16 x i16>, <16 x i16> } poison, <16 x i16> [[TMP8]], 0
-; NEON-NEXT: [[TMP12:%.*]] = insertvalue { <16 x i16>, <16 x i16> } [[TMP11]], <16 x i16> [[TMP10]], 1
-; NEON-NEXT: [[EXTRACT1:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[TMP12]], 0
-; NEON-NEXT: [[EXTRACT2:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[TMP12]], 1
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @deinterleave_wide_i16_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[LOAD:%.*]] = load <32 x i16>, ptr [[PTR]], align 2
-; SVE-FIXED-NEXT: [[DEINTERLEAVE:%.*]] = tail call { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16> [[LOAD]])
-; SVE-FIXED-NEXT: [[EXTRACT1:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[DEINTERLEAVE]], 0
-; SVE-FIXED-NEXT: [[EXTRACT2:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[DEINTERLEAVE]], 1
-; SVE-FIXED-NEXT: ret void
+; NEON-LABEL: deinterleave_wide_i16_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; SVE-FIXED-LABEL: deinterleave_wide_i16_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ret
%load = load <32 x i16>, ptr %ptr, align 2
%deinterleave = tail call { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16> %load)
%extract1 = extractvalue { <16 x i16>, <16 x i16> } %deinterleave, 0
@@ -323,32 +328,48 @@ define void @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
}
define void @interleave_wide_ptr_factor2(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
-; NEON-LABEL: define void @interleave_wide_ptr_factor2
-; NEON-SAME: (ptr [[PTR:%.*]], <8 x ptr> [[L:%.*]], <8 x ptr> [[R:%.*]]) {
-; NEON-NEXT: [[TMP1:%.*]] = getelementptr <2 x ptr>, ptr [[PTR]], i64 0
-; NEON-NEXT: [[TMP2:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[L]], i64 0)
-; NEON-NEXT: [[TMP3:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[R]], i64 0)
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2p0.p0(<2 x ptr> [[TMP2]], <2 x ptr> [[TMP3]], ptr [[TMP1]])
-; NEON-NEXT: [[TMP4:%.*]] = getelementptr <2 x ptr>, ptr [[PTR]], i64 2
-; NEON-NEXT: [[TMP5:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[L]], i64 2)
-; NEON-NEXT: [[TMP6:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[R]], i64 2)
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2p0.p0(<2 x ptr> [[TMP5]], <2 x ptr> [[TMP6]], ptr [[TMP4]])
-; NEON-NEXT: [[TMP7:%.*]] = getelementptr <2 x ptr>, ptr [[PTR]], i64 4
-; NEON-NEXT: [[TMP8:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[L]], i64 4)
-; NEON-NEXT: [[TMP9:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[R]], i64 4)
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2p0.p0(<2 x ptr> [[TMP8]], <2 x ptr> [[TMP9]], ptr [[TMP7]])
-; NEON-NEXT: [[TMP10:%.*]] = getelementptr <2 x ptr>, ptr [[PTR]], i64 6
-; NEON-NEXT: [[TMP11:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[L]], i64 6)
-; NEON-NEXT: [[TMP12:%.*]] = call <2 x ptr> @llvm.vector.extract.v2p0.v8p0(<8 x ptr> [[R]], i64 6)
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2p0.p0(<2 x ptr> [[TMP11]], <2 x ptr> [[TMP12]], ptr [[TMP10]])
-; NEON-NEXT: ret void
-;
-; SVE-FIXED-LABEL: define void @interleave_wide_ptr_factor2
-; SVE-FIXED-SAME: (ptr [[PTR:%.*]], <8 x ptr> [[L:%.*]], <8 x ptr> [[R:%.*]]) #[[ATTR0]] {
-; SVE-FIXED-NEXT: [[INTERLEAVE:%.*]] = tail call <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr> [[L]], <8 x ptr> [[R]])
-; SVE-FIXED-NEXT: store <16 x ptr> [[INTERLEAVE]], ptr [[PTR]], align 4
-; SVE-FIXED-NEXT: ret void
-;
+; NEON-LABEL: interleave_wide_ptr_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q5 killed $q5 killed $q4_q5 def $q4_q5
+; NEON-NEXT: mov v19.16b, v4.16b
+; NEON-NEXT: // kill: def $q7 killed $q7 killed $q6_q7 def $q6_q7
+; NEON-NEXT: mov v17.16b, v6.16b
+; NEON-NEXT: mov x8, x0
+; NEON-NEXT: mov v18.16b, v0.16b
+; NEON-NEXT: mov v4.16b, v1.16b
+; NEON-NEXT: mov v16.16b, v2.16b
+; NEON-NEXT: mov v6.16b, v3.16b
+; NEON-NEXT: st2 { v18.2d, v19.2d }, [x8], #32
+; NEON-NEXT: st2 { v4.2d, v5.2d }, [x8]
+; NEON-NEXT: add x8, x0, #64
+; NEON-NEXT: st2 { v16.2d, v17.2d }, [x8]
+; NEON-NEXT: add x8, x0, #96
+; NEON-NEXT: st2 { v6.2d, v7.2d }, [x8]
+; NEON-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_wide_ptr_factor2:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-FIXED-NEXT: // kill: def $q7 killed $q7 def $z7
+; SVE-FIXED-NEXT: trn2 z16.d, z3.d, z7.d
+; SVE-FIXED-NEXT: // kill: def $q6 killed $q6 def $z6
+; SVE-FIXED-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q5 killed $q5 def $z5
+; SVE-FIXED-NEXT: // kill: def $q4 killed $q4 def $z4
+; SVE-FIXED-NEXT: zip1 z3.d, z3.d, z7.d
+; SVE-FIXED-NEXT: trn2 z7.d, z2.d, z6.d
+; SVE-FIXED-NEXT: zip1 z2.d, z2.d, z6.d
+; SVE-FIXED-NEXT: trn2 z6.d, z1.d, z5.d
+; SVE-FIXED-NEXT: zip1 z1.d, z1.d, z5.d
+; SVE-FIXED-NEXT: stp q2, q7, [x0, #64]
+; SVE-FIXED-NEXT: stp q1, q6, [x0, #32]
+; SVE-FIXED-NEXT: stp q3, q16, [x0, #96]
+; SVE-FIXED-NEXT: trn2 z3.d, z0.d, z4.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z4.d
+; SVE-FIXED-NEXT: stp q0, q3, [x0]
+; SVE-FIXED-NEXT: ret
%interleave = tail call <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr> %l, <8 x ptr> %r)
store <16 x ptr> %interleave, ptr %ptr, align 4
ret void
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
index e48dc5d3051bf..54b8be535d301 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
@@ -1,21 +1,13 @@
-; RUN: opt < %s -interleaved-access -S | FileCheck %s
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define void @extract_user_basic(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_basic(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; CHECK-NEXT: br i1 %c, label %if.then, label %if.merge
-; CHECK: if.then:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: br label %if.merge
-; CHECK: if.merge:
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: extract_user_basic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -30,18 +22,9 @@ if.merge:
}
define void @extract_user_multi(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_multi(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; CHECK-NEXT: br i1 %c, label %if.then, label %if.merge
-; CHECK: if.then:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: br label %if.merge
-; CHECK: if.merge:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: extract_user_multi:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -57,10 +40,9 @@ if.merge:
}
define void @extract_user_multi_no_dom(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_multi_no_dom(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_multi_no_dom:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%e0 = extractelement <8 x i32> %interleaved.vec, i32 0
@@ -76,10 +58,9 @@ if.merge:
}
define void @extract_user_wrong_const_index(ptr %ptr) {
-; CHECK-LABEL: @extract_user_wrong_const_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_wrong_const_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -88,10 +69,9 @@ entry:
}
define void @extract_user_undef_index(ptr %ptr) {
-; CHECK-LABEL: @extract_user_undef_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_undef_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -100,10 +80,9 @@ entry:
}
define void @extract_user_var_index(ptr %ptr, i32 %i) {
-; CHECK-LABEL: @extract_user_var_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_var_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
index ea33590cb241b..95529b023f6d7 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
@@ -1,21 +1,13 @@
-; RUN: opt < %s -interleaved-access -S | FileCheck %s
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck %s
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define void @extract_user_basic(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_basic(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; CHECK-NEXT: br i1 %c, label %if.then, label %if.merge
-; CHECK: if.then:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: br label %if.merge
-; CHECK: if.merge:
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: extract_user_basic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -30,18 +22,9 @@ if.merge:
}
define void @extract_user_multi(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_multi(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; CHECK-NEXT: br i1 %c, label %if.then, label %if.merge
-; CHECK: if.then:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: br label %if.merge
-; CHECK: if.merge:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: extract_user_multi:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -57,10 +40,9 @@ if.merge:
}
define void @extract_user_multi_no_dom(ptr %ptr, i1 %c) {
-; CHECK-LABEL: @extract_user_multi_no_dom(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_multi_no_dom:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%e0 = extractelement <8 x i32> %interleaved.vec, i32 0
@@ -76,10 +58,9 @@ if.merge:
}
define void @extract_user_wrong_const_index(ptr %ptr) {
-; CHECK-LABEL: @extract_user_wrong_const_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_wrong_const_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -88,10 +69,9 @@ entry:
}
define void @extract_user_undef_index(ptr %ptr) {
-; CHECK-LABEL: @extract_user_undef_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_undef_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -100,10 +80,9 @@ entry:
}
define void @extract_user_var_index(ptr %ptr, i32 %i) {
-; CHECK-LABEL: @extract_user_var_index(
-; CHECK-NOT: @llvm.aarch64.neon
-; CHECK: ret void
-;
+; CHECK-LABEL: extract_user_var_index:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
entry:
%interleaved.vec = load <8 x i32>, ptr %ptr, align 8
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index 14986d9eb85c5..1d3094c7d1a80 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -1,21 +1,18 @@
-; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
-; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON
-; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck %s --check-prefix=NEON
+; RUN: llc < %s -mattr=-neon | FileCheck %s --check-prefix=NO_NEON
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define void @load_factor2(ptr %ptr) {
-; NEON-LABEL: @load_factor2(
-; NEON-NEXT: [[LDN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[LDN]], 1
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i8>, ptr %ptr, align 4
%v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -23,16 +20,13 @@ define void @load_factor2(ptr %ptr) {
}
define void @load_factor3(ptr %ptr) {
-; NEON-LABEL: @load_factor3(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -41,17 +35,13 @@ define void @load_factor3(ptr %ptr) {
}
define void @load_factor4(ptr %ptr) {
-; NEON-LABEL: @load_factor4(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -61,31 +51,79 @@ define void @load_factor4(ptr %ptr) {
}
define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
-; NEON-LABEL: @store_factor2(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; NEON-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; NEON-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldrb w8, [sp, #64]
+; NO_NEON-NEXT: ldrb w9, [sp]
+; NO_NEON-NEXT: ldrb w10, [sp, #56]
+; NO_NEON-NEXT: strb w7, [x0, #12]
+; NO_NEON-NEXT: strb w8, [x0, #15]
+; NO_NEON-NEXT: ldrb w8, [sp, #48]
+; NO_NEON-NEXT: strb w9, [x0, #14]
+; NO_NEON-NEXT: ldrb w9, [sp, #40]
+; NO_NEON-NEXT: strb w8, [x0, #11]
+; NO_NEON-NEXT: ldrb w8, [sp, #32]
+; NO_NEON-NEXT: strb w9, [x0, #9]
+; NO_NEON-NEXT: ldrb w9, [sp, #24]
+; NO_NEON-NEXT: strb w8, [x0, #7]
+; NO_NEON-NEXT: ldrb w8, [sp, #16]
+; NO_NEON-NEXT: strb w9, [x0, #5]
+; NO_NEON-NEXT: ldrb w9, [sp, #8]
+; NO_NEON-NEXT: strb w10, [x0, #13]
+; NO_NEON-NEXT: strb w6, [x0, #10]
+; NO_NEON-NEXT: strb w5, [x0, #8]
+; NO_NEON-NEXT: strb w4, [x0, #6]
+; NO_NEON-NEXT: strb w3, [x0, #4]
+; NO_NEON-NEXT: strb w8, [x0, #3]
+; NO_NEON-NEXT: strb w2, [x0, #2]
+; NO_NEON-NEXT: strb w9, [x0, #1]
+; NO_NEON-NEXT: strb w1, [x0]
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i8> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL: @store_factor3(
-; NEON: [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w12, w3
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: ldr w11, [sp, #16]
+; NO_NEON-NEXT: mov w13, w6
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: // kill: def $w7 killed $w7 def $x7
+; NO_NEON-NEXT: // kill: def $w5 killed $w5 def $x5
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: orr x12, x12, x7, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x4, lsl #32
+; NO_NEON-NEXT: orr x9, x9, x2, lsl #32
+; NO_NEON-NEXT: stp x10, x8, [x0, #32]
+; NO_NEON-NEXT: mov w10, w1
+; NO_NEON-NEXT: orr x8, x13, x11, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x5, lsl #32
+; NO_NEON-NEXT: stp x8, x12, [x0, #16]
+; NO_NEON-NEXT: stp x10, x9, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -94,17 +132,38 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
}
define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL: @store_factor4(
-; NEON: [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: ldr w10, [sp]
+; NO_NEON-NEXT: str w7, [x0, #36]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: stp w8, w4, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w9, [x0, #40]
+; NO_NEON-NEXT: ldr w9, [sp, #16]
+; NO_NEON-NEXT: stp w8, w3, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #40]
+; NO_NEON-NEXT: str w9, [x0, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: str w6, [x0, #20]
+; NO_NEON-NEXT: stp w8, w2, [x0, #12]
+; NO_NEON-NEXT: stp w5, w9, [x0, #4]
+; NO_NEON-NEXT: str w1, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -113,17 +172,13 @@ define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
}
define void @load_ptrvec_factor2(ptr %ptr) {
-; NEON-LABEL: @load_ptrvec_factor2(
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
-; NEON-NEXT: [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
-; NEON-NEXT: [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x ptr>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_ptrvec_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_ptrvec_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_ptrvec_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <4 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 1, i32 3>
@@ -131,19 +186,13 @@ define void @load_ptrvec_factor2(ptr %ptr) {
}
define void @load_ptrvec_factor3(ptr %ptr) {
-; NEON-LABEL: @load_ptrvec_factor3(
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 2
-; NEON-NEXT: [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
-; NEON-NEXT: [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x ptr>
-; NEON-NEXT: [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
-; NEON-NEXT: [[TMP7:%.*]] = inttoptr <2 x i64> [[TMP6]] to <2 x ptr>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_ptrvec_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_ptrvec_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_ptrvec_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <6 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 0, i32 3>
%v1 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 1, i32 4>
@@ -152,21 +201,13 @@ define void @load_ptrvec_factor3(ptr %ptr) {
}
define void @load_ptrvec_factor4(ptr %ptr) {
-; NEON-LABEL: @load_ptrvec_factor4(
-; NEON-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld4.v2i64.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 3
-; NEON-NEXT: [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 2
-; NEON-NEXT: [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x ptr>
-; NEON-NEXT: [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
-; NEON-NEXT: [[TMP7:%.*]] = inttoptr <2 x i64> [[TMP6]] to <2 x ptr>
-; NEON-NEXT: [[TMP8:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
-; NEON-NEXT: [[TMP9:%.*]] = inttoptr <2 x i64> [[TMP8]] to <2 x ptr>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_ptrvec_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_ptrvec_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_ptrvec_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 0, i32 4>
%v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 1, i32 5>
@@ -176,35 +217,38 @@ define void @load_ptrvec_factor4(ptr %ptr) {
}
define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
-; NEON-LABEL: @store_ptrvec_factor2(
-; NEON-NEXT: [[TMP1:%.*]] = ptrtoint <2 x ptr> %v0 to <2 x i64>
-; NEON-NEXT: [[TMP2:%.*]] = ptrtoint <2 x ptr> %v1 to <2 x i64>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_ptrvec_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_ptrvec_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_ptrvec_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: stp x3, x5, [x0, #16]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
store <4 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2) {
-; NEON-LABEL: @store_ptrvec_factor3(
-; NEON: [[TMP1:%.*]] = ptrtoint <4 x ptr> %s0 to <4 x i64>
-; NEON-NEXT: [[TMP2:%.*]] = ptrtoint <4 x ptr> %s1 to <4 x i64>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_ptrvec_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_ptrvec_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_ptrvec_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: stp x5, x7, [x0, #32]
+; NO_NEON-NEXT: stp x6, x3, [x0, #16]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
%interleaved.vec = shufflevector <4 x ptr> %s0, <4 x ptr> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
@@ -213,19 +257,24 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
}
define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2, <2 x ptr> %v3) {
-; NEON-LABEL: @store_ptrvec_factor4(
-; NEON: [[TMP1:%.*]] = ptrtoint <4 x ptr> %s0 to <4 x i64>
-; NEON-NEXT: [[TMP2:%.*]] = ptrtoint <4 x ptr> %s1 to <4 x i64>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
-; NEON-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 6, i32 7>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_ptrvec_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_ptrvec_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_ptrvec_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr x8, [sp, #8]
+; NO_NEON-NEXT: stp x3, x5, [x0, #32]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: stp x7, x8, [x0, #48]
+; NO_NEON-NEXT: ldr x8, [sp]
+; NO_NEON-NEXT: stp x6, x8, [x0, #16]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%interleaved.vec = shufflevector <4 x ptr> %s0, <4 x ptr> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
@@ -234,15 +283,13 @@ define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
}
define void @load_undef_mask_factor2(ptr %ptr) {
-; NEON-LABEL: @load_undef_mask_factor2(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_undef_mask_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_undef_mask_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_undef_mask_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <8 x i32>, ptr %ptr, align 4
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
%v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
@@ -250,16 +297,13 @@ define void @load_undef_mask_factor2(ptr %ptr) {
}
define void @load_undef_mask_factor3(ptr %ptr) {
-; NEON-LABEL: @load_undef_mask_factor3(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_undef_mask_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_undef_mask_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_undef_mask_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -268,17 +312,13 @@ define void @load_undef_mask_factor3(ptr %ptr) {
}
define void @load_undef_mask_factor4(ptr %ptr) {
-; NEON-LABEL: @load_undef_mask_factor4(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_undef_mask_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_undef_mask_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_undef_mask_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
@@ -288,31 +328,55 @@ define void @load_undef_mask_factor4(ptr %ptr) {
}
define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
-; NEON-LABEL: @store_undef_mask_factor2(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_undef_mask_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_undef_mask_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_undef_mask_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: stp w3, w7, [x0, #16]
+; NO_NEON-NEXT: stp w4, w8, [x0, #24]
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL: @store_undef_mask_factor3(
-; NEON: [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_undef_mask_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_undef_mask_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_undef_mask_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w11, w3
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: // kill: def $w7 killed $w7 def $x7
+; NO_NEON-NEXT: // kill: def $w5 killed $w5 def $x5
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #16]
+; NO_NEON-NEXT: orr x10, x10, x4, lsl #32
+; NO_NEON-NEXT: lsl x9, x9, #32
+; NO_NEON-NEXT: stp x10, x8, [x0, #32]
+; NO_NEON-NEXT: orr x8, x11, x7, lsl #32
+; NO_NEON-NEXT: mov w10, w1
+; NO_NEON-NEXT: lsl x11, x2, #32
+; NO_NEON-NEXT: stp x9, x8, [x0, #16]
+; NO_NEON-NEXT: orr x9, x10, x5, lsl #32
+; NO_NEON-NEXT: stp x9, x11, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -321,17 +385,35 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL: @store_undef_mask_factor4(
-; NEON: [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_undef_mask_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_undef_mask_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_undef_mask_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: ldr w10, [sp]
+; NO_NEON-NEXT: str w7, [x0, #36]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: stp w8, w4, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w9, [x0, #40]
+; NO_NEON-NEXT: ldr w9, [sp, #16]
+; NO_NEON-NEXT: stp w8, w3, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #8]
+; NO_NEON-NEXT: stp w6, w9, [x0, #20]
+; NO_NEON-NEXT: stp w1, w5, [x0]
+; NO_NEON-NEXT: str w8, [x0, #8]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -340,185 +422,399 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @load_illegal_factor2(ptr %ptr) nounwind {
-; NEON-LABEL: @load_illegal_factor2(
-; NEON-NOT: @llvm.aarch64.neon
-; NEON: ret void
-; NO_NEON-LABEL: @load_illegal_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_illegal_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_illegal_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <3 x float>, ptr %ptr, align 16
%v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
ret void
}
define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
-; NEON-LABEL: @store_illegal_factor2(
-; NEON-NOT: @llvm.aarch64.neon
-; NEON: ret void
-; NO_NEON-LABEL: @store_illegal_factor2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_illegal_factor2:
+; NEON: // %bb.0:
+; NEON-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; NEON-NEXT: str d0, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_illegal_factor2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: fmov w8, s0
+; NO_NEON-NEXT: fmov w9, s2
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: str x8, [x0]
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <3 x float> %v0, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
store <3 x float> %interleaved.vec, ptr %ptr, align 16
ret void
}
define void @store_general_mask_factor4(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor4(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor4(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor4:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor4:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor4_undefbeg(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefbeg(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor4_undefbeg:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor4_undefbeg:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: str w10, [x0, #4]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor4_undefend(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefend(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor4_undefend:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor4_undefend:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #96]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #216]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: str w8, [x0, #20]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: str w9, [x0, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #88]
+; NO_NEON-NEXT: stp w10, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w9, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor4_undefmid(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefmid(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor4_undefmid:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor4_undefmid:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #96]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: ldr w10, [sp, #216]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: str w8, [x0, #20]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: str w9, [x0, #28]
+; NO_NEON-NEXT: stp w10, w8, [x0, #8]
+; NO_NEON-NEXT: str w5, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor4_undefmulti(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor4_undefmulti(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor4_undefmulti:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v5.16b, v2.16b
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: mov v3.16b, v0.16b
+; NEON-NEXT: fmov d4, d3
+; NEON-NEXT: st4 { v2.2s, v3.2s, v4.2s, v5.2s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor4_undefmulti:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: str w5, [x0]
+; NO_NEON-NEXT: str w8, [x0, #12]
+; NO_NEON-NEXT: str w9, [x0, #28]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor3(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor3(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor3:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w10, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w6 killed $w6 def $x6
+; NO_NEON-NEXT: ldr w11, [sp, #224]
+; NO_NEON-NEXT: ldr w12, [sp, #96]
+; NO_NEON-NEXT: ldr w9, [sp, #104]
+; NO_NEON-NEXT: ldr w13, [sp, #16]
+; NO_NEON-NEXT: orr x8, x8, x10, lsl #32
+; NO_NEON-NEXT: orr x10, x11, x12, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp, #232]
+; NO_NEON-NEXT: orr x9, x9, x13, lsl #32
+; NO_NEON-NEXT: mov w13, w7
+; NO_NEON-NEXT: ldr w12, [sp, #88]
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: orr x11, x13, x11, lsl #32
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x12, x12, x6, lsl #32
+; NO_NEON-NEXT: stp x11, x9, [x0, #24]
+; NO_NEON-NEXT: mov w9, w5
+; NO_NEON-NEXT: orr x8, x9, x8, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #8]
+; NO_NEON-NEXT: str x8, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor3_undefmultimid(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_undefmultimid(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor3_undefmultimid:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor3_undefmultimid:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: ldr w10, [sp, #104]
+; NO_NEON-NEXT: ldr w11, [sp, #16]
+; NO_NEON-NEXT: ldr w12, [sp, #232]
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: orr x10, x10, x11, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp, #96]
+; NO_NEON-NEXT: lsl x12, x12, #32
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x9, x9, x11, lsl #32
+; NO_NEON-NEXT: mov w11, w5
+; NO_NEON-NEXT: str x10, [x0, #32]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: orr x8, x11, x8, lsl #32
+; NO_NEON-NEXT: stp x9, x12, [x0, #16]
+; NO_NEON-NEXT: stp x8, x10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor3_undef_fail(
-; NEON-NOT: @llvm.aarch64.neon
-; NEON: ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_undef_fail(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor3_undef_fail:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [sp]
+; NEON-NEXT: trn2 v3.4s, v0.4s, v4.4s
+; NEON-NEXT: uzp2 v5.4s, v0.4s, v4.4s
+; NEON-NEXT: zip1 v1.4s, v1.4s, v0.4s
+; NEON-NEXT: mov v3.s[0], v4.s[2]
+; NEON-NEXT: uzp1 v0.4s, v5.4s, v0.4s
+; NEON-NEXT: mov v1.s[2], v4.s[0]
+; NEON-NEXT: mov v3.s[1], v2.s[0]
+; NEON-NEXT: stp q1, q0, [x0]
+; NEON-NEXT: str q3, [x0, #32]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor3_undef_fail:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: ldr w10, [sp, #104]
+; NO_NEON-NEXT: ldr w11, [sp, #24]
+; NO_NEON-NEXT: ldr w12, [sp, #232]
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: orr x10, x10, x11, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp, #96]
+; NO_NEON-NEXT: lsl x12, x12, #32
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x9, x9, x11, lsl #32
+; NO_NEON-NEXT: mov w11, w5
+; NO_NEON-NEXT: str x10, [x0, #32]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: orr x8, x11, x8, lsl #32
+; NO_NEON-NEXT: stp x9, x12, [x0, #16]
+; NO_NEON-NEXT: stp x8, x10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor3_undeflane(
-; NEON-NEXT: [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_undeflane(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor3_undeflane:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v0.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor3_undeflane:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: ldr w10, [sp, #104]
+; NO_NEON-NEXT: ldr w11, [sp, #232]
+; NO_NEON-NEXT: ldr w12, [sp, #224]
+; NO_NEON-NEXT: ldr w13, [sp, #96]
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: str x10, [x0, #32]
+; NO_NEON-NEXT: ldr w10, [sp, #216]
+; NO_NEON-NEXT: lsl x9, x11, #32
+; NO_NEON-NEXT: orr x11, x12, x13, lsl #32
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #88]
+; NO_NEON-NEXT: stp x11, x9, [x0, #16]
+; NO_NEON-NEXT: lsl x9, x10, #32
+; NO_NEON-NEXT: stp x9, x8, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_general_mask_factor3_negativestart(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: @store_general_mask_factor3_negativestart(
-; NEON-NOT: @llvm.aarch64.neon
-; NEON: ret void
-; NO_NEON-LABEL: @store_general_mask_factor3_negativestart(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_general_mask_factor3_negativestart:
+; NEON: // %bb.0:
+; NEON-NEXT: ldr q1, [sp]
+; NEON-NEXT: trn2 v2.4s, v1.4s, v4.4s
+; NEON-NEXT: uzp2 v3.4s, v1.4s, v4.4s
+; NEON-NEXT: trn1 v5.4s, v1.4s, v4.4s
+; NEON-NEXT: mov v2.s[0], v4.s[2]
+; NEON-NEXT: uzp1 v3.4s, v3.4s, v1.4s
+; NEON-NEXT: ext v1.16b, v1.16b, v5.16b, #12
+; NEON-NEXT: mov v2.s[1], v0.s[2]
+; NEON-NEXT: stp q1, q3, [x0]
+; NEON-NEXT: str q2, [x0, #32]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_general_mask_factor3_negativestart:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w3 killed $w3 def $x3
+; NO_NEON-NEXT: ldr w10, [sp, #104]
+; NO_NEON-NEXT: ldr w11, [sp, #232]
+; NO_NEON-NEXT: ldr w12, [sp, #96]
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: orr x10, x10, x3, lsl #32
+; NO_NEON-NEXT: lsl x11, x11, #32
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x9, x9, x12, lsl #32
+; NO_NEON-NEXT: str x10, [x0, #32]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: lsl x8, x8, #32
+; NO_NEON-NEXT: stp x9, x11, [x0, #16]
+; NO_NEON-NEXT: stp x8, x10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -527,37 +823,36 @@ define void @store_general_mask_factor3_negativestart(ptr %ptr, <32 x i32> %v0,
@g = external global <4 x float>
; The following does not give a valid interleaved store
-; NEON-LABEL: define void @no_interleave
-; NEON-NOT: call void @llvm.aarch64.neon.st2
-; NEON: shufflevector
-; NEON: store
-; NEON: ret void
-; NO_NEON-LABEL: define void @no_interleave
-; NO_NEON: shufflevector
-; NO_NEON: store
-; NO_NEON: ret void
define void @no_interleave(<4 x float> %a0) {
+; NEON-LABEL: no_interleave:
+; NEON: // %bb.0:
+; NEON-NEXT: dup v1.4s, v0.s[3]
+; NEON-NEXT: adrp x8, :got:g
+; NEON-NEXT: ldr x8, [x8, :got_lo12:g]
+; NEON-NEXT: mov v1.s[0], v0.s[0]
+; NEON-NEXT: str q1, [x8]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: no_interleave:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: adrp x8, :got:g
+; NO_NEON-NEXT: ldr x8, [x8, :got_lo12:g]
+; NO_NEON-NEXT: stp s0, s3, [x8]
+; NO_NEON-NEXT: str s3, [x8, #8]
+; NO_NEON-NEXT: ret
%v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 3, i32 7, i32 undef>
store <4 x float> %v0, ptr @g, align 16
ret void
}
define void @load_factor2_wide2(ptr %ptr) {
-; NEON-LABEL: @load_factor2_wide2(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr %ptr, i32 8
-; NEON-NEXT: [[LDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[TMP5]])
-; NEON-NEXT: [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 1
-; NEON-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 0
-; NEON-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor2_wide2(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor2_wide2:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor2_wide2:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -565,29 +860,13 @@ define void @load_factor2_wide2(ptr %ptr) {
}
define void @load_factor2_wide3(ptr %ptr) {
-; NEON-LABEL: @load_factor2_wide3(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[PTR:%.*]])
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[PTR]], i32 8
-; NEON-NEXT: [[LDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[TMP5]])
-; NEON-NEXT: [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 1
-; NEON-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 0
-; NEON-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[TMP5]], i32 8
-; NEON-NEXT: [[LDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[TMP9]])
-; NEON-NEXT: [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 1
-; NEON-NEXT: [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 0
-; NEON-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; NEON-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; NEON-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor2_wide3(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor2_wide3:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor2_wide3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
@@ -595,24 +874,13 @@ define void @load_factor2_wide3(ptr %ptr) {
}
define void @load_factor3_wide(ptr %ptr) {
-; NEON-LABEL: @load_factor3_wide(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr %ptr, i32 12
-; NEON-NEXT: [[LDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0(ptr [[TMP6]])
-; NEON-NEXT: [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 2
-; NEON-NEXT: [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 1
-; NEON-NEXT: [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 0
-; NEON-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor3_wide(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor3_wide:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor3_wide:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
@@ -621,27 +889,13 @@ define void @load_factor3_wide(ptr %ptr) {
}
define void @load_factor4_wide(ptr %ptr) {
-; NEON-LABEL: @load_factor4_wide(
-; NEON-NEXT: [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0(ptr %ptr)
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
-; NEON-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
-; NEON-NEXT: [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
-; NEON-NEXT: [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
-; NEON-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr %ptr, i32 16
-; NEON-NEXT: [[LDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0(ptr [[TMP7]])
-; NEON-NEXT: [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 3
-; NEON-NEXT: [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 2
-; NEON-NEXT: [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 1
-; NEON-NEXT: [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 0
-; NEON-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @load_factor4_wide(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor4_wide:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor4_wide:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <32 x i32>, ptr %ptr, align 4
%v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%v1 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
@@ -651,38 +905,102 @@ define void @load_factor4_wide(ptr %ptr) {
}
define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
-; NEON-LABEL: @store_factor2_wide(
-; NEON-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], ptr %ptr)
-; NEON-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr %ptr, i32 8
-; NEON-NEXT: call void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32> [[TMP5]], <4 x i32> [[TMP6]], ptr [[TMP7]])
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor2_wide(
-; NO_NEON: ret void
+; NEON-LABEL: store_factor2_wide:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v5.16b, v2.16b
+; NEON-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; NEON-NEXT: mov v4.16b, v0.16b
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; NEON-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor2_wide:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp]
+; NO_NEON-NEXT: ldr w10, [sp, #56]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: ldr w9, [sp, #40]
+; NO_NEON-NEXT: stp w7, w10, [x0, #48]
+; NO_NEON-NEXT: stp w6, w8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: stp w5, w9, [x0, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: stp w4, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #16]
+; NO_NEON-NEXT: stp w3, w9, [x0, #16]
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: stp w2, w8, [x0, #8]
+; NO_NEON-NEXT: stp w1, w9, [x0]
+; NO_NEON-NEXT: ret
%interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-LABEL: @store_factor3_wide(
-; NEON: [[TMP2:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], ptr %ptr)
-; NEON-NEXT: [[TMP6:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; NEON-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr %ptr, i32 12
-; NEON-NEXT: call void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], ptr [[TMP9]])
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor3_wide(
-; NO_NEON: ret void
+; NEON-LABEL: store_factor3_wide:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v4.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor3_wide:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w10, [sp, #48]
+; NO_NEON-NEXT: ldr w11, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w6 killed $w6 def $x6
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w18, w7
+; NO_NEON-NEXT: ldr w9, [sp, #64]
+; NO_NEON-NEXT: ldr w8, [sp, #120]
+; NO_NEON-NEXT: mov w17, w5
+; NO_NEON-NEXT: ldr w14, [sp, #128]
+; NO_NEON-NEXT: orr x10, x10, x11, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp]
+; NO_NEON-NEXT: ldr w12, [sp, #104]
+; NO_NEON-NEXT: ldr w13, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: ldr w15, [sp, #88]
+; NO_NEON-NEXT: orr x9, x9, x14, lsl #32
+; NO_NEON-NEXT: orr x8, x8, x11, lsl #32
+; NO_NEON-NEXT: ldr w16, [sp, #16]
+; NO_NEON-NEXT: ldr w11, [sp, #56]
+; NO_NEON-NEXT: orr x12, x12, x6, lsl #32
+; NO_NEON-NEXT: ldr w14, [sp, #40]
+; NO_NEON-NEXT: str x9, [x0, #88]
+; NO_NEON-NEXT: ldr w9, [sp, #96]
+; NO_NEON-NEXT: str x8, [x0, #80]
+; NO_NEON-NEXT: ldr w8, [sp, #80]
+; NO_NEON-NEXT: orr x11, x18, x11, lsl #32
+; NO_NEON-NEXT: orr x9, x13, x9, lsl #32
+; NO_NEON-NEXT: orr x13, x15, x4, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #56]
+; NO_NEON-NEXT: orr x8, x16, x8, lsl #32
+; NO_NEON-NEXT: ldr w12, [sp, #24]
+; NO_NEON-NEXT: str x11, [x0, #72]
+; NO_NEON-NEXT: stp x13, x9, [x0, #32]
+; NO_NEON-NEXT: ldr w13, [sp, #72]
+; NO_NEON-NEXT: mov w9, w3
+; NO_NEON-NEXT: str x8, [x0, #16]
+; NO_NEON-NEXT: ldr w8, [sp, #8]
+; NO_NEON-NEXT: mov w11, w1
+; NO_NEON-NEXT: orr x10, x17, x14, lsl #32
+; NO_NEON-NEXT: orr x9, x9, x12, lsl #32
+; NO_NEON-NEXT: orr x12, x13, x2, lsl #32
+; NO_NEON-NEXT: orr x8, x11, x8, lsl #32
+; NO_NEON-NEXT: str x10, [x0, #48]
+; NO_NEON-NEXT: str x9, [x0, #24]
+; NO_NEON-NEXT: stp x8, x12, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
@@ -691,23 +1009,72 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
}
define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-LABEL: @store_factor4_wide(
-; NEON: [[TMP2:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; NEON-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
-; NEON-NEXT: [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], ptr %ptr)
-; NEON-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; NEON-NEXT: [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
-; NEON-NEXT: [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
-; NEON-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr %ptr, i32 16
-; NEON-NEXT: call void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], ptr [[TMP11]])
-; NEON-NEXT: ret void
-; NO_NEON-LABEL: @store_factor4_wide(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: store_factor4_wide:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-NEXT: mov v19.16b, v6.16b
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: mov v6.16b, v5.16b
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v5.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v4.16b, v1.16b
+; NEON-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: store_factor4_wide:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #192]
+; NO_NEON-NEXT: ldr w9, [sp, #128]
+; NO_NEON-NEXT: ldr w10, [sp, #64]
+; NO_NEON-NEXT: str w7, [x0, #96]
+; NO_NEON-NEXT: str w8, [x0, #124]
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: str w9, [x0, #120]
+; NO_NEON-NEXT: ldr w9, [sp, #184]
+; NO_NEON-NEXT: str w10, [x0, #116]
+; NO_NEON-NEXT: ldr w10, [sp, #120]
+; NO_NEON-NEXT: str w8, [x0, #112]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: stp w10, w9, [x0, #104]
+; NO_NEON-NEXT: ldr w9, [sp, #176]
+; NO_NEON-NEXT: ldr w10, [sp, #48]
+; NO_NEON-NEXT: str w8, [x0, #100]
+; NO_NEON-NEXT: ldr w8, [sp, #112]
+; NO_NEON-NEXT: str w10, [x0, #84]
+; NO_NEON-NEXT: ldr w10, [sp, #40]
+; NO_NEON-NEXT: stp w8, w9, [x0, #88]
+; NO_NEON-NEXT: ldr w8, [sp, #168]
+; NO_NEON-NEXT: ldr w9, [sp, #104]
+; NO_NEON-NEXT: str w10, [x0, #68]
+; NO_NEON-NEXT: ldr w10, [sp, #32]
+; NO_NEON-NEXT: stp w9, w8, [x0, #72]
+; NO_NEON-NEXT: ldr w8, [sp, #160]
+; NO_NEON-NEXT: ldr w9, [sp, #96]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #152]
+; NO_NEON-NEXT: ldr w9, [sp, #88]
+; NO_NEON-NEXT: str w10, [x0, #36]
+; NO_NEON-NEXT: ldr w10, [sp, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #144]
+; NO_NEON-NEXT: ldr w9, [sp, #80]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #8]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #136]
+; NO_NEON-NEXT: ldr w9, [sp, #72]
+; NO_NEON-NEXT: str w6, [x0, #80]
+; NO_NEON-NEXT: str w5, [x0, #64]
+; NO_NEON-NEXT: str w4, [x0, #48]
+; NO_NEON-NEXT: str w3, [x0, #32]
+; NO_NEON-NEXT: str w2, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w1, w10, [x0]
+; NO_NEON-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
@@ -716,13 +1083,13 @@ define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
}
define void @load_factor2_fp128(ptr %ptr) {
-; NEON-LABEL: @load_factor2_fp128(
-; NEON-NOT: @llvm.aarch64.neon
-; NEON: ret void
-; NO_NEON-LABEL: @load_factor2_fp128(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret void
+; NEON-LABEL: load_factor2_fp128:
+; NEON: // %bb.0:
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_factor2_fp128:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ret
%interleaved.vec = load <4 x fp128>, ptr %ptr, align 16
%v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 1, i32 3>
@@ -730,23 +1097,32 @@ define void @load_factor2_fp128(ptr %ptr) {
}
define <4 x i1> @load_large_vector(ptr %p) {
-; NEON-LABEL: @load_large_vector(
-; NEON: [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0(ptr
-; NEON-NEXT: [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
-; NEON-NEXT: [[TMP2:%.*]] = inttoptr <2 x i64> [[TMP1]] to <2 x ptr>
-; NEON-NEXT: [[TMP3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
-; NEON-NEXT: [[TMP4:%.*]] = inttoptr <2 x i64> [[TMP3]] to <2 x ptr>
-; NEON: [[LDN1:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0(ptr
-; NEON-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN1]], 1
-; NEON-NEXT: [[TMP6:%.*]] = inttoptr <2 x i64> [[TMP5]] to <2 x ptr>
-; NEON-NEXT: [[TMP7:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN1]], 0
-; NEON-NEXT: [[TMP8:%.*]] = inttoptr <2 x i64> [[TMP7]] to <2 x ptr>
-; NEON-NEXT: shufflevector <2 x ptr> [[TMP2]], <2 x ptr> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NEON-NEXT: shufflevector <2 x ptr> [[TMP4]], <2 x ptr> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; NO_NEON-LABEL: @load_large_vector(
-; NO_NEON-NOT: @llvm.aarch64.neon
-; NO_NEON: ret
+; NEON-LABEL: load_large_vector:
+; NEON: // %bb.0:
+; NEON-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; NEON-NEXT: ld3 { v3.2d, v4.2d, v5.2d }, [x0]
+; NEON-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; NEON-NEXT: cmeq v1.2d, v3.2d, v4.2d
+; NEON-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; NEON-NEXT: mvn v0.16b, v0.16b
+; NEON-NEXT: xtn v0.4h, v0.4s
+; NEON-NEXT: ret
;
+; NO_NEON-LABEL: load_large_vector:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x9, x8, [x0]
+; NO_NEON-NEXT: ldp x11, x10, [x0, #24]
+; NO_NEON-NEXT: ldp x13, x12, [x0, #48]
+; NO_NEON-NEXT: cmp x9, x8
+; NO_NEON-NEXT: ldp x9, x8, [x0, #72]
+; NO_NEON-NEXT: cset w0, ne
+; NO_NEON-NEXT: cmp x11, x10
+; NO_NEON-NEXT: cset w1, ne
+; NO_NEON-NEXT: cmp x13, x12
+; NO_NEON-NEXT: cset w2, ne
+; NO_NEON-NEXT: cmp x9, x8
+; NO_NEON-NEXT: cset w3, ne
+; NO_NEON-NEXT: ret
%l = load <12 x ptr>, ptr %p
%s1 = shufflevector <12 x ptr> %l, <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%s2 = shufflevector <12 x ptr> %l, <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index 7c676875acf92..7fda8d70e3e97 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1,19 +1,12 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -interleaved-access -S | FileCheck %s
-; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s | FileCheck %s
target triple = "aarch64-linux-gnu"
define void @load_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16.p0(<vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP4]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <32 x i16>, ptr %ptr, align 4
%v0 = shufflevector <32 x i16> %interleaved.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14,
i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -23,17 +16,9 @@ define void @load_factor2(ptr %ptr) #0 {
}
define void @load_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32.p0(<vscale x 4 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP6]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
@@ -42,19 +27,9 @@ define void @load_factor3(ptr %ptr) #0 {
}
define void @load_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <16 x i64>, ptr %ptr, align 4
%v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -64,15 +39,18 @@ define void @load_factor4(ptr %ptr) #0 {
}
define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
-; CHECK-LABEL: define void @store_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]], <16 x i16> [[V0:%.*]], <16 x i16> [[V1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> poison, <16 x i16> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> poison, <16 x i16> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv8i16.p0(<vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <16 x i16> %v0, <16 x i16> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x i16> %interleaved.vec, ptr %ptr, align 4
@@ -80,19 +58,21 @@ define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
}
define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
-; CHECK-LABEL: define void @store_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <8 x i32> [[V2:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[V2]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP6]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st3.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -104,21 +84,24 @@ define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2
}
define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
-; CHECK-LABEL: define void @store_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]], <4 x i64> [[V2:%.*]], <4 x i64> [[V3:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i64> [[V2]], <4 x i64> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st4.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -127,17 +110,9 @@ define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2
}
define void @load_ptrvec_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_ptrvec_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT: [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_ptrvec_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -145,20 +120,9 @@ define void @load_ptrvec_factor2(ptr %ptr) #0 {
}
define void @load_ptrvec_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_ptrvec_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT: [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: [[TMP10:%.*]] = inttoptr <4 x i64> [[TMP9]] to <4 x ptr>
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_ptrvec_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <12 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -167,23 +131,9 @@ define void @load_ptrvec_factor3(ptr %ptr) #0 {
}
define void @load_ptrvec_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_ptrvec_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT: [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: [[TMP10:%.*]] = inttoptr <4 x i64> [[TMP9]] to <4 x ptr>
-; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP12:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP11]], i64 0)
-; CHECK-NEXT: [[TMP13:%.*]] = inttoptr <4 x i64> [[TMP12]] to <4 x ptr>
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_ptrvec_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <16 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -193,38 +143,39 @@ define void @load_ptrvec_factor4(ptr %ptr) #0 {
}
define void @store_ptrvec_factor2(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
-; CHECK-LABEL: define void @store_ptrvec_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint <4 x ptr> [[V0]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint <4 x ptr> [[V1]] to <4 x i64>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_ptrvec_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2) #0 {
-; CHECK-LABEL: define void @store_ptrvec_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]], <4 x ptr> [[V2:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x ptr> [[V0]], <4 x ptr> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x ptr> [[V2]], <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint <8 x ptr> [[S0]] to <8 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st3.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_ptrvec_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st3d { z16.d - z18.d }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x ptr> %v2, <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -233,23 +184,24 @@ define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
}
define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
-; CHECK-LABEL: define void @store_ptrvec_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]], <4 x ptr> [[V2:%.*]], <4 x ptr> [[V3:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x ptr> [[V0]], <4 x ptr> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x ptr> [[V2]], <4 x ptr> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint <8 x ptr> [[S0]] to <8 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP10]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st4.nxv2i64.p0(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_ptrvec_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x ptr> %v2, <4 x ptr> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13,
@@ -259,23 +211,9 @@ define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
}
define void @load_factor2_wide(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_factor2_wide(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[PTR]], i32 8
-; CHECK-NEXT: [[LDN1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[TMP6]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN1]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP7]], i64 0)
-; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN1]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP9]], i64 0)
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_factor2_wide:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <16 x i64>, ptr %ptr, align 4
%v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -283,21 +221,25 @@ define void @load_factor2_wide(ptr %ptr) #0 {
}
define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
-; CHECK-LABEL: define void @store_factor2_wide(
-; CHECK-SAME: ptr [[PTR:%.*]], <8 x i64> [[V0:%.*]], <8 x i64> [[V1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[PTR]], i32 8
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv2i64.p0(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> splat (i1 true), ptr [[TMP10]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_factor2_wide:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v17.16b, v4.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z5.d
+; CHECK-NEXT: // kill: def $q6 killed $q6 def $z5_z6
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: st2d { z16.d, z17.d }, p1, [x0]
+; CHECK-NEXT: splice z6.d, p0, z6.d, z7.d
+; CHECK-NEXT: st2d { z5.d, z6.d }, p1, [x0, #2, mul vl]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <8 x i64> %v0, <8 x i64> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -305,27 +247,9 @@ define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
; Check that neon is used for illegal multiples of 128-bit types
define void @load_384bit(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_384bit(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[PTR]], i32 4
-; CHECK-NEXT: [[LDN1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[TMP3]])
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN1]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN1]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP3]], i32 4
-; CHECK-NEXT: [[LDN2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[TMP6]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN2]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN2]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP10]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_384bit:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <12 x i64>, ptr %ptr, align 4
%v0 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10>
%v1 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
@@ -334,13 +258,9 @@ define void @load_384bit(ptr %ptr) #0 {
; Check that neon is used for 128-bit vectors
define void @load_128bit(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_128bit(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_128bit:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <4 x i64>, ptr %ptr, align 4
%v0 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
@@ -349,16 +269,9 @@ define void @load_128bit(ptr %ptr) #0 {
; Check that correct ptrues are generated for min != max case
define void @load_min_not_max(ptr %ptr) #1 {
-; CHECK-LABEL: define void @load_min_not_max(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_min_not_max:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <8 x i64>, ptr %ptr, align 4
%v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -366,16 +279,18 @@ define void @load_min_not_max(ptr %ptr) #1 {
}
define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
-; CHECK-LABEL: define void @store_min_not_max(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_min_not_max:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -383,16 +298,9 @@ define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
; Check that correct ptrues are generated for min > type case
define void @load_min_ge_type(ptr %ptr) #2 {
-; CHECK-LABEL: define void @load_min_ge_type(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64.p0(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_min_ge_type:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <8 x i64>, ptr %ptr, align 4
%v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -400,35 +308,27 @@ define void @load_min_ge_type(ptr %ptr) #2 {
}
define void @store_min_ge_type(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
-; CHECK-LABEL: define void @store_min_ge_type(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_min_ge_type:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @load_double_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_double_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 2
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP8]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_double_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <16 x double>, ptr %ptr, align 4
%v0 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -438,17 +338,9 @@ define void @load_double_factor4(ptr %ptr) #0 {
}
define void @load_float_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_float_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32.p0(<vscale x 4 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP6]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_float_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <24 x float>, ptr %ptr, align 4
%v0 = shufflevector <24 x float> %interleaved.vec, <24 x float> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x float> %interleaved.vec, <24 x float> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
@@ -457,15 +349,9 @@ define void @load_float_factor3(ptr %ptr) #0 {
}
define void @load_half_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_half_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16.p0(<vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP4]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_half_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <32 x half>, ptr %ptr, align 4
%v0 = shufflevector <32 x half> %interleaved.vec, <32 x half> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%v1 = shufflevector <32 x half> %interleaved.vec, <32 x half> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
@@ -473,15 +359,9 @@ define void @load_half_factor2(ptr %ptr) #0 {
}
define void @load_bfloat_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define void @load_bfloat_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[LDN:%.*]] = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16.p0(<vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP4]], i64 0)
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: load_bfloat_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%interleaved.vec = load <32 x bfloat>, ptr %ptr, align 4
%v0 = shufflevector <32 x bfloat> %interleaved.vec, <32 x bfloat> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%v1 = shufflevector <32 x bfloat> %interleaved.vec, <32 x bfloat> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
@@ -489,21 +369,24 @@ define void @load_bfloat_factor2(ptr %ptr) #0 {
}
define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
-; CHECK-LABEL: define void @store_double_factor4(
-; CHECK-SAME: ptr [[PTR:%.*]], <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x double> [[V0]], <4 x double> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x double> [[V2]], <4 x double> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP6]], i64 0)
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP8]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st4.nxv2f64.p0(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP7]], <vscale x 2 x double> [[TMP9]], <vscale x 2 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_double_factor4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -512,19 +395,21 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
}
define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8 x float> %v2) #0 {
-; CHECK-LABEL: define void @store_float_factor3(
-; CHECK-SAME: ptr [[PTR:%.*]], <8 x float> [[V0:%.*]], <8 x float> [[V1:%.*]], <8 x float> [[V2:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[S0:%.*]] = shufflevector <8 x float> [[V0]], <8 x float> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[S1:%.*]] = shufflevector <8 x float> [[V2]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP6]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st3.nxv4f32.p0(<vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP7]], <vscale x 4 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_float_factor3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
%s0 = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x float> %v2, <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -536,15 +421,18 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
}
define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
-; CHECK-LABEL: define void @store_half_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]], <16 x half> [[V0:%.*]], <16 x half> [[V1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> poison, <16 x half> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> poison, <16 x half> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv8f16.p0(<vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_half_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v5.16b, v2.16b
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v4.16b, v0.16b
+; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <16 x half> %v0, <16 x half> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x half> %interleaved.vec, ptr %ptr, align 4
@@ -553,15 +441,26 @@ define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1) #0 {
-; CHECK-LABEL: define void @store_bfloat_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]], <16 x bfloat> [[V0:%.*]], <16 x bfloat> [[V1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> poison, <16 x bfloat> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> poison, <16 x bfloat> [[TMP4]], i64 0)
-; CHECK-NEXT: call void @llvm.aarch64.sve.st2.nxv8bf16.p0(<vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x i1> splat (i1 true), ptr [[PTR]])
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: store_bfloat_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-2
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: str z0, [sp]
+; CHECK-NEXT: addvl x8, sp, #1
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: str q1, [sp, #16]
+; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: str z2, [sp, #1, mul vl]
+; CHECK-NEXT: str q3, [x8, #16]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: addvl sp, sp, #2
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%interleaved.vec = shufflevector <16 x bfloat> %v0, <16 x bfloat> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x bfloat> %interleaved.vec, ptr %ptr, align 4
@@ -570,26 +469,9 @@ define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1
; Ensure vscale_range property does not affect scalable vector types.
define void @deinterleave_nxptr_factor2(ptr %ptr) #2 {
-; CHECK-LABEL: define void @deinterleave_nxptr_factor2(
-; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 0
-; CHECK-NEXT: [[LDN1:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[TMP1]])
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP4]], i64 0)
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[LDN2:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64.p0(<vscale x 2 x i1> splat (i1 true), ptr [[TMP6]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN2]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP7]], i64 2)
-; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN2]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP5]], <vscale x 2 x double> [[TMP9]], i64 2)
-; CHECK-NEXT: [[TMP11:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } poison, <vscale x 4 x double> [[TMP8]], 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP11]], <vscale x 4 x double> [[TMP10]], 1
-; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP12]], 0
-; CHECK-NEXT: [[EXTRACT2:%.*]] = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP12]], 1
-; CHECK-NEXT: ret void
-;
+; CHECK-LABEL: deinterleave_nxptr_factor2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ret
%wide.vec = load <vscale x 8 x double>, ptr %ptr, align 8
%ldN = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec)
%extract1 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %ldN, 0
>From 4366e426c1ace1e4cc69825802759986b80f00ea Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 29 Jul 2026 10:24:20 +0000
Subject: [PATCH 3/5] [AArch64] Generate test for enabled and disabled IA
---
.../AArch64/binopshuffles-inseltpoison.ll | 139 +++-
llvm/test/CodeGen/AArch64/binopshuffles.ll | 137 +++-
.../AArch64/fixed-deinterleave-intrinsics.ll | 194 +++--
...aved-accesses-extract-user-inseltpoison.ll | 6 +-
.../interleaved-accesses-extract-user.ll | 6 +-
.../CodeGen/AArch64/interleaved-accesses.ll | 641 +++++++++++----
.../AArch64/sve-interleaved-accesses.ll | 775 +++++++++++++-----
7 files changed, 1398 insertions(+), 500 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
index 7f7e6ce967b01..fff83e14ec55a 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
@@ -1,16 +1,25 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define <4 x float> @vld2(ptr %pSrc) {
-; CHECK-LABEL: vld2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld2:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -22,13 +31,34 @@ entry:
}
define <4 x float> @vld3(ptr %pSrc) {
-; CHECK-LABEL: vld3:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld3:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld3:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
@@ -43,12 +73,30 @@ entry:
}
define <4 x float> @vld4(ptr %pSrc) {
-; CHECK-LABEL: vld4:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld4:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v3.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmul v2.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: trn1 v5.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v5.d[1]
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
@@ -65,13 +113,22 @@ entry:
}
define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: twosrc:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: twosrc:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x1]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -84,13 +141,25 @@ entry:
}
define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: twosrc2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: twosrc2:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc2:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q2, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q4, [x1]
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v4.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v5.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v3.4s, v4.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -102,3 +171,5 @@ entry:
%l8 = fadd fast <4 x float> %l6, %l5
ret <4 x float> %l8
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 519fed793603e..66fa65ffeaecf 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -1,16 +1,25 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
define <4 x float> @vld2(ptr %pSrc) {
-; CHECK-LABEL: vld2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld2:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -22,13 +31,34 @@ entry:
}
define <4 x float> @vld3(ptr %pSrc) {
-; CHECK-LABEL: vld3:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld3:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld3:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
+; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
@@ -43,12 +73,30 @@ entry:
}
define <4 x float> @vld4(ptr %pSrc) {
-; CHECK-LABEL: vld4:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: vld4:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld4:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v3.4s, v3.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmul v2.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: trn1 v5.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: zip2 v2.4s, v2.4s, v3.4s
+; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v5.d[1]
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
@@ -65,13 +113,22 @@ entry:
}
define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: twosrc:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: twosrc:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
+; CHECK-IADISABLED-NEXT: ldp q2, q3, [x1]
+; CHECK-IADISABLED-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -84,13 +141,25 @@ entry:
}
define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-LABEL: twosrc2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: twosrc2:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc2:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q2, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q4, [x1]
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v4.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v5.4s, v3.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v3.4s, v4.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
diff --git a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
index 3f5d44af1d1e0..610e579a8f94b 100644
--- a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc < %s | FileCheck %s --check-prefix=NEON
-; RUN: llc < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefix=SVE-FIXED
+; RUN: llc < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
+; RUN: llc < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IADISABLED
target triple = "aarch64-linux-gnu"
@@ -110,12 +112,19 @@ define void @deinterleave_ptr_factor2(ptr %ptr) {
}
define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
-; NEON-LABEL: interleave_i8_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.16b, v1.16b }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_i8_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i8_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
+; NEON-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i8_factor2:
; SVE-FIXED: // %bb.0:
@@ -161,12 +170,19 @@ define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
}
define void @interleave_i16_factor2(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
-; NEON-LABEL: interleave_i16_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.8h, v1.8h }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_i16_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i16_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; NEON-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i16_factor2:
; SVE-FIXED: // %bb.0:
@@ -196,12 +212,19 @@ define void @interleave_i16_factor2(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
}
define void @interleave_i32_factor2(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
-; NEON-LABEL: interleave_i32_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_i32_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i32_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i32_factor2:
; SVE-FIXED: // %bb.0:
@@ -223,12 +246,19 @@ define void @interleave_i32_factor2(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
}
define void @interleave_i64_factor2(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
-; NEON-LABEL: interleave_i64_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_i64_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i64_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_i64_factor2:
; SVE-FIXED: // %bb.0:
@@ -244,12 +274,19 @@ define void @interleave_i64_factor2(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
}
define void @interleave_float_factor2(ptr %ptr, <4 x float> %l, <4 x float> %r) {
-; NEON-LABEL: interleave_float_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_float_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_float_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_float_factor2:
; SVE-FIXED: // %bb.0:
@@ -271,12 +308,19 @@ define void @interleave_float_factor2(ptr %ptr, <4 x float> %l, <4 x float> %r)
}
define void @interleave_double_factor2(ptr %ptr, <2 x double> %l, <2 x double> %r) {
-; NEON-LABEL: interleave_double_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_double_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_double_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_double_factor2:
; SVE-FIXED: // %bb.0:
@@ -292,12 +336,19 @@ define void @interleave_double_factor2(ptr %ptr, <2 x double> %l, <2 x double> %
}
define void @interleave_ptr_factor2(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
-; NEON-LABEL: interleave_ptr_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_ptr_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_ptr_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_ptr_factor2:
; SVE-FIXED: // %bb.0:
@@ -328,24 +379,40 @@ define void @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
}
define void @interleave_wide_ptr_factor2(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
-; NEON-LABEL: interleave_wide_ptr_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q5 killed $q5 killed $q4_q5 def $q4_q5
-; NEON-NEXT: mov v19.16b, v4.16b
-; NEON-NEXT: // kill: def $q7 killed $q7 killed $q6_q7 def $q6_q7
-; NEON-NEXT: mov v17.16b, v6.16b
-; NEON-NEXT: mov x8, x0
-; NEON-NEXT: mov v18.16b, v0.16b
-; NEON-NEXT: mov v4.16b, v1.16b
-; NEON-NEXT: mov v16.16b, v2.16b
-; NEON-NEXT: mov v6.16b, v3.16b
-; NEON-NEXT: st2 { v18.2d, v19.2d }, [x8], #32
-; NEON-NEXT: st2 { v4.2d, v5.2d }, [x8]
-; NEON-NEXT: add x8, x0, #64
-; NEON-NEXT: st2 { v16.2d, v17.2d }, [x8]
-; NEON-NEXT: add x8, x0, #96
-; NEON-NEXT: st2 { v6.2d, v7.2d }, [x8]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: interleave_wide_ptr_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q4_q5 def $q4_q5
+; NEON-IAENABLED-NEXT: mov v19.16b, v4.16b
+; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q6_q7 def $q6_q7
+; NEON-IAENABLED-NEXT: mov v17.16b, v6.16b
+; NEON-IAENABLED-NEXT: mov x8, x0
+; NEON-IAENABLED-NEXT: mov v18.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v6.16b, v3.16b
+; NEON-IAENABLED-NEXT: st2 { v18.2d, v19.2d }, [x8], #32
+; NEON-IAENABLED-NEXT: st2 { v4.2d, v5.2d }, [x8]
+; NEON-IAENABLED-NEXT: add x8, x0, #64
+; NEON-IAENABLED-NEXT: st2 { v16.2d, v17.2d }, [x8]
+; NEON-IAENABLED-NEXT: add x8, x0, #96
+; NEON-IAENABLED-NEXT: st2 { v6.2d, v7.2d }, [x8]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_wide_ptr_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v16.2d, v3.2d, v7.2d
+; NEON-IADISABLED-NEXT: zip1 v3.2d, v3.2d, v7.2d
+; NEON-IADISABLED-NEXT: zip2 v7.2d, v2.2d, v6.2d
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v6.2d
+; NEON-IADISABLED-NEXT: zip2 v6.2d, v1.2d, v5.2d
+; NEON-IADISABLED-NEXT: zip1 v1.2d, v1.2d, v5.2d
+; NEON-IADISABLED-NEXT: stp q3, q16, [x0, #96]
+; NEON-IADISABLED-NEXT: zip2 v3.2d, v0.2d, v4.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v4.2d
+; NEON-IADISABLED-NEXT: stp q1, q6, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q2, q7, [x0, #64]
+; NEON-IADISABLED-NEXT: stp q0, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: interleave_wide_ptr_factor2:
; SVE-FIXED: // %bb.0:
@@ -392,3 +459,6 @@ declare <8 x float> @llvm.vector.interleave2.v8f32(<4 x float>, <4 x float>)
declare <4 x double> @llvm.vector.interleave2.v4f64(<2 x double>, <2 x double>)
declare <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr>, <2 x ptr>)
declare <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr>, <8 x ptr>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SVE-FIXED-IADISABLED: {{.*}}
+; SVE-FIXED-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
index 54b8be535d301..bdb64c98d33e4 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
@@ -89,3 +90,6 @@ entry:
%e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-IADISABLED: {{.*}}
+; CHECK-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
index 95529b023f6d7..463ad52a3196d 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
@@ -89,3 +90,6 @@ entry:
%e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-IADISABLED: {{.*}}
+; CHECK-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index 1d3094c7d1a80..ba6699b1a5837 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s --check-prefix=NEON
-; RUN: llc < %s -mattr=-neon | FileCheck %s --check-prefix=NO_NEON
+; RUN: llc < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
+; RUN: llc < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IADISABLED
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
@@ -51,12 +53,20 @@ define void @load_factor4(ptr %ptr) {
}
define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
-; NEON-LABEL: store_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; NEON-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; NEON-NEXT: st2 { v0.8b, v1.8b }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; NEON-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; NEON-IAENABLED-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; NEON-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; NEON-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; NEON-IADISABLED-NEXT: str q0, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor2:
; NO_NEON: // %bb.0:
@@ -92,13 +102,29 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
}
define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL: store_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: trn2 v3.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: trn1 v5.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: ext v3.16b, v3.16b, v0.16b, #4
+; NEON-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v4.4s
+; NEON-IADISABLED-NEXT: mov v5.d[1], v0.d[0]
+; NEON-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; NEON-IADISABLED-NEXT: mov v1.s[1], v2.s[1]
+; NEON-IADISABLED-NEXT: mov v5.s[2], v2.s[0]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; NEON-IADISABLED-NEXT: stp q5, q1, [x0]
+; NEON-IADISABLED-NEXT: str q3, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor3:
; NO_NEON: // %bb.0:
@@ -132,14 +158,33 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
}
define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL: store_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v5.4s, v2.4s, v3.4s
+; NEON-IADISABLED-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip2 v3.4s, v2.4s, v3.4s
+; NEON-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: uzp2 v4.4s, v4.4s, v0.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: ext v1.16b, v2.16b, v5.16b, #8
+; NEON-IADISABLED-NEXT: mov v7.d[1], v6.d[1]
+; NEON-IADISABLED-NEXT: mov v16.d[1], v5.d[1]
+; NEON-IADISABLED-NEXT: mov v4.d[1], v3.d[1]
+; NEON-IADISABLED-NEXT: mov v0.d[1], v1.d[1]
+; NEON-IADISABLED-NEXT: stp q0, q16, [x0]
+; NEON-IADISABLED-NEXT: stp q7, q4, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor4:
; NO_NEON: // %bb.0:
@@ -217,12 +262,19 @@ define void @load_ptrvec_factor4(ptr %ptr) {
}
define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
-; NEON-LABEL: store_ptrvec_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.2d, v1.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_ptrvec_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_ptrvec_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_ptrvec_factor2:
; NO_NEON: // %bb.0:
@@ -235,13 +287,22 @@ define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
}
define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2) {
-; NEON-LABEL: store_ptrvec_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_ptrvec_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_ptrvec_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v3.2d, v1.2d, v2.2d
+; NEON-IADISABLED-NEXT: zip1 v1.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: mov v0.d[0], v2.d[0]
+; NEON-IADISABLED-NEXT: stp q0, q3, [x0, #16]
+; NEON-IADISABLED-NEXT: str q1, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_ptrvec_factor3:
; NO_NEON: // %bb.0:
@@ -257,14 +318,24 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
}
define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2, <2 x ptr> %v3) {
-; NEON-LABEL: store_ptrvec_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_ptrvec_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_ptrvec_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v4.2d, v2.2d, v3.2d
+; NEON-IADISABLED-NEXT: zip2 v5.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v3.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q5, q4, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_ptrvec_factor4:
; NO_NEON: // %bb.0:
@@ -328,12 +399,18 @@ define void @load_undef_mask_factor4(ptr %ptr) {
}
define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
-; NEON-LABEL: store_undef_mask_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; NEON-NEXT: st2 { v0.4s, v1.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_undef_mask_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_undef_mask_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: str q0, [x0, #16]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_undef_mask_factor2:
; NO_NEON: // %bb.0:
@@ -347,13 +424,27 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
}
define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
-; NEON-LABEL: store_undef_mask_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_undef_mask_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-IAENABLED-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_undef_mask_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: trn2 v3.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: trn1 v1.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: ext v3.16b, v3.16b, v0.16b, #4
+; NEON-IADISABLED-NEXT: mov v4.16b, v1.16b
+; NEON-IADISABLED-NEXT: mov v1.s[1], v2.s[1]
+; NEON-IADISABLED-NEXT: mov v4.d[1], v0.d[0]
+; NEON-IADISABLED-NEXT: mov v3.s[0], v2.s[2]
+; NEON-IADISABLED-NEXT: stp q4, q1, [x0]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v2.s[3]
+; NEON-IADISABLED-NEXT: str q3, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_undef_mask_factor3:
; NO_NEON: // %bb.0:
@@ -385,14 +476,31 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
-; NEON-LABEL: store_undef_mask_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_undef_mask_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_undef_mask_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip2 v5.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v6.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: mov v1.s[2], v2.s[1]
+; NEON-IADISABLED-NEXT: trn1 v7.4s, v2.4s, v3.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
+; NEON-IADISABLED-NEXT: zip2 v4.4s, v2.4s, v3.4s
+; NEON-IADISABLED-NEXT: mov v6.s[2], v2.s[0]
+; NEON-IADISABLED-NEXT: mov v1.s[3], v3.s[1]
+; NEON-IADISABLED-NEXT: mov v5.d[1], v7.d[1]
+; NEON-IADISABLED-NEXT: mov v0.d[1], v4.d[1]
+; NEON-IADISABLED-NEXT: stp q6, q1, [x0]
+; NEON-IADISABLED-NEXT: stp q5, q0, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_undef_mask_factor4:
; NO_NEON: // %bb.0:
@@ -454,14 +562,26 @@ define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
}
define void @store_general_mask_factor4(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
-; NEON-NEXT: mov v6.16b, v2.16b
-; NEON-NEXT: mov v3.16b, v1.16b
-; NEON-NEXT: ldr d5, [sp]
-; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-IAENABLED-NEXT: mov v6.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr d5, [sp]
+; NEON-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [sp]
+; NEON-IADISABLED-NEXT: trn2 v3.4s, v1.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: ext v0.16b, v0.16b, v2.16b, #8
+; NEON-IADISABLED-NEXT: mov v3.d[1], v2.d[1]
+; NEON-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; NEON-IADISABLED-NEXT: stp q1, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor4:
; NO_NEON: // %bb.0:
@@ -487,14 +607,29 @@ define void @store_general_mask_factor4(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1
}
define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor4_undefbeg:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
-; NEON-NEXT: mov v6.16b, v2.16b
-; NEON-NEXT: mov v3.16b, v1.16b
-; NEON-NEXT: ldr d5, [sp]
-; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor4_undefbeg:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-IAENABLED-NEXT: mov v6.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr d5, [sp]
+; NEON-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor4_undefbeg:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v2.4s, v1.4s
+; NEON-IADISABLED-NEXT: mov v5.16b, v4.16b
+; NEON-IADISABLED-NEXT: ldr q1, [sp]
+; NEON-IADISABLED-NEXT: trn1 v3.4s, v4.4s, v1.4s
+; NEON-IADISABLED-NEXT: ext v0.16b, v0.16b, v2.16b, #8
+; NEON-IADISABLED-NEXT: ext v3.16b, v4.16b, v3.16b, #12
+; NEON-IADISABLED-NEXT: mov v5.s[0], v0.s[0]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v2.s[0]
+; NEON-IADISABLED-NEXT: mov v5.s[2], v1.s[1]
+; NEON-IADISABLED-NEXT: mov v5.s[3], v0.s[3]
+; NEON-IADISABLED-NEXT: stp q3, q5, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor4_undefbeg:
; NO_NEON: // %bb.0:
@@ -520,14 +655,26 @@ define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x
}
define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor4_undefend:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
-; NEON-NEXT: mov v6.16b, v2.16b
-; NEON-NEXT: mov v3.16b, v1.16b
-; NEON-NEXT: ldr d5, [sp]
-; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor4_undefend:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-IAENABLED-NEXT: mov v6.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr d5, [sp]
+; NEON-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor4_undefend:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [sp]
+; NEON-IADISABLED-NEXT: trn2 v3.4s, v1.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v2.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: mov v3.s[2], v0.s[1]
+; NEON-IADISABLED-NEXT: ext v2.16b, v0.16b, v2.16b, #8
+; NEON-IADISABLED-NEXT: mov v1.d[1], v2.d[1]
+; NEON-IADISABLED-NEXT: stp q1, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor4_undefend:
; NO_NEON: // %bb.0:
@@ -552,14 +699,25 @@ define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x
}
define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor4_undefmid:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
-; NEON-NEXT: mov v6.16b, v2.16b
-; NEON-NEXT: mov v3.16b, v1.16b
-; NEON-NEXT: ldr d5, [sp]
-; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor4_undefmid:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-IAENABLED-NEXT: mov v6.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr d5, [sp]
+; NEON-IAENABLED-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor4_undefmid:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v1.4s, v2.4s
+; NEON-IADISABLED-NEXT: ldr q3, [sp]
+; NEON-IADISABLED-NEXT: uzp1 v3.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: mov v3.s[3], v2.s[0]
+; NEON-IADISABLED-NEXT: mov v0.s[1], v4.s[1]
+; NEON-IADISABLED-NEXT: stp q3, q0, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor4_undefmid:
; NO_NEON: // %bb.0:
@@ -583,14 +741,21 @@ define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x
}
define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor4_undefmulti:
-; NEON: // %bb.0:
-; NEON-NEXT: mov v5.16b, v2.16b
-; NEON-NEXT: mov v2.16b, v1.16b
-; NEON-NEXT: mov v3.16b, v0.16b
-; NEON-NEXT: fmov d4, d3
-; NEON-NEXT: st4 { v2.2s, v3.2s, v4.2s, v5.2s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor4_undefmulti:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: mov v5.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v0.16b
+; NEON-IAENABLED-NEXT: fmov d4, d3
+; NEON-IAENABLED-NEXT: st4 { v2.2s, v3.2s, v4.2s, v5.2s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor4_undefmulti:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: mov v1.s[3], v2.s[0]
+; NEON-IADISABLED-NEXT: dup v0.4s, v2.s[1]
+; NEON-IADISABLED-NEXT: stp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor4_undefmulti:
; NO_NEON: // %bb.0:
@@ -610,13 +775,30 @@ define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32
}
define void @store_general_mask_factor3(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
-; NEON-NEXT: mov v2.16b, v1.16b
-; NEON-NEXT: ldr q3, [sp]
-; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-IAENABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr q3, [sp]
+; NEON-IAENABLED-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [sp]
+; NEON-IADISABLED-NEXT: trn2 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: trn1 v5.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: ext v2.16b, v2.16b, v1.16b, #4
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v0.4s, v3.4s
+; NEON-IADISABLED-NEXT: mov v5.d[1], v1.d[0]
+; NEON-IADISABLED-NEXT: mov v2.s[0], v4.s[2]
+; NEON-IADISABLED-NEXT: mov v0.s[1], v4.s[1]
+; NEON-IADISABLED-NEXT: mov v5.s[2], v4.s[0]
+; NEON-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; NEON-IADISABLED-NEXT: stp q5, q0, [x0]
+; NEON-IADISABLED-NEXT: str q2, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor3:
; NO_NEON: // %bb.0:
@@ -653,13 +835,28 @@ define void @store_general_mask_factor3(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1
}
define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor3_undefmultimid:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
-; NEON-NEXT: mov v2.16b, v1.16b
-; NEON-NEXT: ldr q3, [sp]
-; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor3_undefmultimid:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-IAENABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IAENABLED-NEXT: ldr q3, [sp]
+; NEON-IAENABLED-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor3_undefmultimid:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [sp]
+; NEON-IADISABLED-NEXT: trn2 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v3.4s, v0.4s, v4.4s
+; NEON-IADISABLED-NEXT: ext v2.16b, v2.16b, v1.16b, #4
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp1 v0.4s, v3.4s, v0.4s
+; NEON-IADISABLED-NEXT: mov v2.s[0], v4.s[2]
+; NEON-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
+; NEON-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; NEON-IADISABLED-NEXT: stp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: str q2, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor3_undefmultimid:
; NO_NEON: // %bb.0:
@@ -739,13 +936,26 @@ define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32
}
define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
-; NEON-LABEL: store_general_mask_factor3_undeflane:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
-; NEON-NEXT: mov v2.16b, v0.16b
-; NEON-NEXT: ldr q3, [sp]
-; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_general_mask_factor3_undeflane:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-IAENABLED-NEXT: mov v2.16b, v0.16b
+; NEON-IAENABLED-NEXT: ldr q3, [sp]
+; NEON-IAENABLED-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_general_mask_factor3_undeflane:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [sp]
+; NEON-IADISABLED-NEXT: trn2 v1.4s, v0.4s, v4.4s
+; NEON-IADISABLED-NEXT: uzp2 v2.4s, v0.4s, v4.4s
+; NEON-IADISABLED-NEXT: ext v3.16b, v4.16b, v0.16b, #4
+; NEON-IADISABLED-NEXT: mov v1.s[0], v4.s[2]
+; NEON-IADISABLED-NEXT: uzp1 v0.4s, v2.4s, v0.4s
+; NEON-IADISABLED-NEXT: ext v2.16b, v3.16b, v4.16b, #8
+; NEON-IADISABLED-NEXT: stp q0, q1, [x0, #16]
+; NEON-IADISABLED-NEXT: str q2, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_general_mask_factor3_undeflane:
; NO_NEON: // %bb.0:
@@ -905,15 +1115,25 @@ define void @load_factor4_wide(ptr %ptr) {
}
define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
-; NEON-LABEL: store_factor2_wide:
-; NEON: // %bb.0:
-; NEON-NEXT: mov v5.16b, v2.16b
-; NEON-NEXT: // kill: def $q3 killed $q3 def $q2_q3
-; NEON-NEXT: mov v4.16b, v0.16b
-; NEON-NEXT: mov v2.16b, v1.16b
-; NEON-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
-; NEON-NEXT: st2 { v2.4s, v3.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor2_wide:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: mov v5.16b, v2.16b
+; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; NEON-IAENABLED-NEXT: mov v4.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; NEON-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor2_wide:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip2 v3.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q0, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor2_wide:
; NO_NEON: // %bb.0:
@@ -941,17 +1161,44 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
}
define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-LABEL: store_factor3_wide:
-; NEON: // %bb.0:
-; NEON-NEXT: mov v18.16b, v4.16b
-; NEON-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-NEXT: mov v17.16b, v2.16b
-; NEON-NEXT: mov v4.16b, v3.16b
-; NEON-NEXT: mov v16.16b, v0.16b
-; NEON-NEXT: mov v3.16b, v1.16b
-; NEON-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
-; NEON-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor3_wide:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
+; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v3.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor3_wide:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: trn2 v6.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: uzp1 v7.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: trn2 v16.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: trn1 v17.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: uzp1 v18.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: ext v6.16b, v6.16b, v1.16b, #4
+; NEON-IADISABLED-NEXT: uzp2 v2.4s, v2.4s, v7.4s
+; NEON-IADISABLED-NEXT: trn1 v7.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: ext v16.16b, v16.16b, v0.16b, #4
+; NEON-IADISABLED-NEXT: mov v17.d[1], v0.d[0]
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v3.4s, v18.4s
+; NEON-IADISABLED-NEXT: mov v6.s[0], v5.s[2]
+; NEON-IADISABLED-NEXT: mov v7.d[1], v1.d[0]
+; NEON-IADISABLED-NEXT: mov v2.s[1], v4.s[1]
+; NEON-IADISABLED-NEXT: mov v16.s[0], v4.s[2]
+; NEON-IADISABLED-NEXT: mov v17.s[2], v4.s[0]
+; NEON-IADISABLED-NEXT: mov v0.s[1], v5.s[1]
+; NEON-IADISABLED-NEXT: mov v6.s[3], v5.s[3]
+; NEON-IADISABLED-NEXT: mov v7.s[2], v5.s[0]
+; NEON-IADISABLED-NEXT: mov v16.s[3], v4.s[3]
+; NEON-IADISABLED-NEXT: stp q17, q2, [x0]
+; NEON-IADISABLED-NEXT: stp q0, q6, [x0, #64]
+; NEON-IADISABLED-NEXT: stp q16, q7, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor3_wide:
; NO_NEON: // %bb.0:
@@ -1009,19 +1256,54 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
}
define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-LABEL: store_factor4_wide:
-; NEON: // %bb.0:
-; NEON-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
-; NEON-NEXT: mov v19.16b, v6.16b
-; NEON-NEXT: mov v18.16b, v4.16b
-; NEON-NEXT: mov v6.16b, v5.16b
-; NEON-NEXT: mov v17.16b, v2.16b
-; NEON-NEXT: mov v5.16b, v3.16b
-; NEON-NEXT: mov v16.16b, v0.16b
-; NEON-NEXT: mov v4.16b, v1.16b
-; NEON-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
-; NEON-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: store_factor4_wide:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-IAENABLED-NEXT: mov v19.16b, v6.16b
+; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
+; NEON-IAENABLED-NEXT: mov v6.16b, v5.16b
+; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v5.16b, v3.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
+; NEON-IAENABLED-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor4_wide:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: uzp2 v16.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip1 v17.4s, v5.4s, v7.4s
+; NEON-IADISABLED-NEXT: trn1 v18.4s, v5.4s, v7.4s
+; NEON-IADISABLED-NEXT: zip2 v19.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: uzp2 v20.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: trn2 v21.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip1 v22.4s, v4.4s, v6.4s
+; NEON-IADISABLED-NEXT: zip2 v7.4s, v5.4s, v7.4s
+; NEON-IADISABLED-NEXT: uzp2 v16.4s, v16.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: ext v3.16b, v5.16b, v17.16b, #8
+; NEON-IADISABLED-NEXT: trn1 v5.4s, v4.4s, v6.4s
+; NEON-IADISABLED-NEXT: mov v19.d[1], v18.d[1]
+; NEON-IADISABLED-NEXT: zip2 v18.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: zip2 v6.4s, v4.4s, v6.4s
+; NEON-IADISABLED-NEXT: uzp2 v20.4s, v20.4s, v0.4s
+; NEON-IADISABLED-NEXT: mov v21.d[1], v17.d[1]
+; NEON-IADISABLED-NEXT: trn2 v17.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: ext v2.16b, v4.16b, v22.16b, #8
+; NEON-IADISABLED-NEXT: mov v16.d[1], v7.d[1]
+; NEON-IADISABLED-NEXT: mov v1.d[1], v3.d[1]
+; NEON-IADISABLED-NEXT: mov v18.d[1], v5.d[1]
+; NEON-IADISABLED-NEXT: stp q21, q19, [x0, #80]
+; NEON-IADISABLED-NEXT: mov v20.d[1], v6.d[1]
+; NEON-IADISABLED-NEXT: mov v17.d[1], v22.d[1]
+; NEON-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
+; NEON-IADISABLED-NEXT: str q16, [x0, #112]
+; NEON-IADISABLED-NEXT: stp q20, q1, [x0, #48]
+; NEON-IADISABLED-NEXT: stp q17, q18, [x0, #16]
+; NEON-IADISABLED-NEXT: str q0, [x0]
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: store_factor4_wide:
; NO_NEON: // %bb.0:
@@ -1097,16 +1379,34 @@ define void @load_factor2_fp128(ptr %ptr) {
}
define <4 x i1> @load_large_vector(ptr %p) {
-; NEON-LABEL: load_large_vector:
-; NEON: // %bb.0:
-; NEON-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0], #48
-; NEON-NEXT: ld3 { v3.2d, v4.2d, v5.2d }, [x0]
-; NEON-NEXT: cmeq v0.2d, v0.2d, v1.2d
-; NEON-NEXT: cmeq v1.2d, v3.2d, v4.2d
-; NEON-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; NEON-NEXT: mvn v0.16b, v0.16b
-; NEON-NEXT: xtn v0.4h, v0.4s
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_large_vector:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; NEON-IAENABLED-NEXT: ld3 { v3.2d, v4.2d, v5.2d }, [x0]
+; NEON-IAENABLED-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; NEON-IAENABLED-NEXT: cmeq v1.2d, v3.2d, v4.2d
+; NEON-IAENABLED-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; NEON-IAENABLED-NEXT: mvn v0.16b, v0.16b
+; NEON-IAENABLED-NEXT: xtn v0.4h, v0.4s
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_large_vector:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q2, [x0, #32]
+; NEON-IADISABLED-NEXT: add x8, x0, #24
+; NEON-IADISABLED-NEXT: ldr q0, [x0, #80]
+; NEON-IADISABLED-NEXT: ldr q3, [x0]
+; NEON-IADISABLED-NEXT: add x9, x0, #72
+; NEON-IADISABLED-NEXT: ext v1.16b, v3.16b, v1.16b, #8
+; NEON-IADISABLED-NEXT: ext v0.16b, v2.16b, v0.16b, #8
+; NEON-IADISABLED-NEXT: ld1 { v3.d }[1], [x8]
+; NEON-IADISABLED-NEXT: ld1 { v2.d }[1], [x9]
+; NEON-IADISABLED-NEXT: cmeq v0.2d, v2.2d, v0.2d
+; NEON-IADISABLED-NEXT: cmeq v1.2d, v3.2d, v1.2d
+; NEON-IADISABLED-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: mvn v0.16b, v0.16b
+; NEON-IADISABLED-NEXT: xtn v0.4h, v0.4s
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_large_vector:
; NO_NEON: // %bb.0:
@@ -1129,3 +1429,6 @@ define <4 x i1> @load_large_vector(ptr %p) {
%ret = icmp ne <4 x ptr> %s1, %s2
ret <4 x i1> %ret
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NO_NEON-IADISABLED: {{.*}}
+; NO_NEON-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index 7fda8d70e3e97..aa9684bef45c1 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
target triple = "aarch64-linux-gnu"
@@ -39,18 +40,36 @@ define void @load_factor4(ptr %ptr) #0 {
}
define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
-; CHECK-LABEL: store_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
-; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.h, vl8
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-IAENABLED-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: st2h { z4.h, z5.h }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.h, vl8
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI3_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI3_0
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-IADISABLED-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-IADISABLED-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-IADISABLED-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-IADISABLED-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <16 x i16> %v0, <16 x i16> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x i16> %interleaved.vec, ptr %ptr, align 4
@@ -58,21 +77,92 @@ define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
}
define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
-; CHECK-LABEL: store_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl4
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-IAENABLED-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-IAENABLED-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-IAENABLED-NEXT: ptrue p0.s
+; CHECK-IAENABLED-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #144
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: add x11, sp, #96
+; CHECK-IADISABLED-NEXT: add x9, sp, #64
+; CHECK-IADISABLED-NEXT: mov w12, #20 // =0x14
+; CHECK-IADISABLED-NEXT: orr x8, x11, #0x1c
+; CHECK-IADISABLED-NEXT: orr x10, x9, #0x10
+; CHECK-IADISABLED-NEXT: mov s6, v4.s[3]
+; CHECK-IADISABLED-NEXT: st1 { v1.s }[1], [x8]
+; CHECK-IADISABLED-NEXT: orr x8, x11, #0x8
+; CHECK-IADISABLED-NEXT: st1 { v2.s }[3], [x8]
+; CHECK-IADISABLED-NEXT: orr x8, x11, #0x4
+; CHECK-IADISABLED-NEXT: st1 { v0.s }[3], [x8]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: st1 { v4.s }[2], [x8]
+; CHECK-IADISABLED-NEXT: st1 { v2.s }[1], [x10]
+; CHECK-IADISABLED-NEXT: orr x10, x9, #0xc
+; CHECK-IADISABLED-NEXT: st1 { v0.s }[1], [x10]
+; CHECK-IADISABLED-NEXT: orr x10, x9, x12
+; CHECK-IADISABLED-NEXT: st1 { v4.s }[1], [x10]
+; CHECK-IADISABLED-NEXT: add x10, sp, #32
+; CHECK-IADISABLED-NEXT: orr x12, x10, #0x10
+; CHECK-IADISABLED-NEXT: st1 { v3.s }[1], [x10]
+; CHECK-IADISABLED-NEXT: st1 { v5.s }[2], [x12]
+; CHECK-IADISABLED-NEXT: orr x12, x10, #0xc
+; CHECK-IADISABLED-NEXT: st1 { v3.s }[2], [x12]
+; CHECK-IADISABLED-NEXT: orr x12, x10, #0x8
+; CHECK-IADISABLED-NEXT: st1 { v1.s }[2], [x12]
+; CHECK-IADISABLED-NEXT: orr x12, x10, #0x4
+; CHECK-IADISABLED-NEXT: st1 { v5.s }[1], [x12]
+; CHECK-IADISABLED-NEXT: stp s1, s3, [sp, #112]
+; CHECK-IADISABLED-NEXT: mov s1, v1.s[3]
+; CHECK-IADISABLED-NEXT: str s5, [sp, #24]
+; CHECK-IADISABLED-NEXT: str s6, [sp, #12]
+; CHECK-IADISABLED-NEXT: ldr z6, [x11]
+; CHECK-IADISABLED-NEXT: stp s0, s2, [sp, #64]
+; CHECK-IADISABLED-NEXT: str s4, [sp, #72]
+; CHECK-IADISABLED-NEXT: str s1, [sp, #52]
+; CHECK-IADISABLED-NEXT: mov z7.s, z6.s[5]
+; CHECK-IADISABLED-NEXT: mov z16.s, z6.s[4]
+; CHECK-IADISABLED-NEXT: mov w12, v6.s[2]
+; CHECK-IADISABLED-NEXT: mov w14, v6.s[1]
+; CHECK-IADISABLED-NEXT: fmov w11, s7
+; CHECK-IADISABLED-NEXT: mov z7.s, z6.s[7]
+; CHECK-IADISABLED-NEXT: mov s6, v2.s[2]
+; CHECK-IADISABLED-NEXT: mov s2, v3.s[3]
+; CHECK-IADISABLED-NEXT: fmov w13, s16
+; CHECK-IADISABLED-NEXT: stp w14, w12, [sp, #4]
+; CHECK-IADISABLED-NEXT: str s7, [sp, #28]
+; CHECK-IADISABLED-NEXT: mov s7, v0.s[2]
+; CHECK-IADISABLED-NEXT: mov s0, v5.s[3]
+; CHECK-IADISABLED-NEXT: stp w13, w11, [sp, #16]
+; CHECK-IADISABLED-NEXT: stp s7, s6, [sp, #88]
+; CHECK-IADISABLED-NEXT: stp s2, s0, [sp, #56]
+; CHECK-IADISABLED-NEXT: ldr z0, [x10]
+; CHECK-IADISABLED-NEXT: ldr z1, [x9]
+; CHECK-IADISABLED-NEXT: ldr z2, [x8]
+; CHECK-IADISABLED-NEXT: str z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0]
+; CHECK-IADISABLED-NEXT: str z2, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -84,24 +174,49 @@ define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2
}
define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
-; CHECK-LABEL: store_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v19.16b, v6.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-IAENABLED-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IADISABLED-NEXT: // kill: def $q4 killed $q4 def $z4
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IADISABLED-NEXT: // kill: def $q6 killed $q6 def $z6
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: splice z5.d, p0, z5.d, z7.d
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z4.d, p0, z4.d, z6.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: uzp2 z2.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp1 z1.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp2 z3.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: uzp1 z0.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: str z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z3, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -143,39 +258,91 @@ define void @load_ptrvec_factor4(ptr %ptr) #0 {
}
define void @store_ptrvec_factor2(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
-; CHECK-LABEL: store_ptrvec_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
-; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_ptrvec_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_ptrvec_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI9_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI9_0
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: tbl z1.d, { z1.d }, z4.d
+; CHECK-IADISABLED-NEXT: tbl z0.d, { z0.d }, z4.d
+; CHECK-IADISABLED-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
}
define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2) #0 {
-; CHECK-LABEL: store_ptrvec_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st3d { z16.d - z18.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_ptrvec_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: st3d { z16.d - z18.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_ptrvec_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #112
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: mov d6, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov x9, sp
+; CHECK-IADISABLED-NEXT: st1 { v2.d }[1], [x8]
+; CHECK-IADISABLED-NEXT: stp d3, d0, [sp, #56]
+; CHECK-IADISABLED-NEXT: mov d0, v4.d[1]
+; CHECK-IADISABLED-NEXT: stp d2, d4, [sp, #72]
+; CHECK-IADISABLED-NEXT: mov d2, v5.d[1]
+; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
+; CHECK-IADISABLED-NEXT: str d6, [sp, #88]
+; CHECK-IADISABLED-NEXT: stp d0, d1, [sp, #40]
+; CHECK-IADISABLED-NEXT: mov d0, v1.d[1]
+; CHECK-IADISABLED-NEXT: stp d4, d2, [sp, #16]
+; CHECK-IADISABLED-NEXT: stp d5, d0, [sp]
+; CHECK-IADISABLED-NEXT: ldr z0, [x9]
+; CHECK-IADISABLED-NEXT: add x9, sp, #64
+; CHECK-IADISABLED-NEXT: ldr z1, [x8]
+; CHECK-IADISABLED-NEXT: ldr z2, [x9]
+; CHECK-IADISABLED-NEXT: str z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z2, [x0]
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x ptr> %v2, <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -184,24 +351,49 @@ define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
}
define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
-; CHECK-LABEL: store_ptrvec_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_ptrvec_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v19.16b, v6.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-IAENABLED-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_ptrvec_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IADISABLED-NEXT: // kill: def $q4 killed $q4 def $z4
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IADISABLED-NEXT: // kill: def $q6 killed $q6 def $z6
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: splice z5.d, p0, z5.d, z7.d
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z4.d, p0, z4.d, z6.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: uzp2 z2.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp1 z1.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp2 z3.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: uzp1 z0.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: str z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z3, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x ptr> %v2, <4 x ptr> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13,
@@ -221,25 +413,53 @@ define void @load_factor2_wide(ptr %ptr) #0 {
}
define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
-; CHECK-LABEL: store_factor2_wide:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v17.16b, v4.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: ptrue p1.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z5.d
-; CHECK-NEXT: // kill: def $q6 killed $q6 def $z5_z6
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
-; CHECK-NEXT: st2d { z16.d, z17.d }, p1, [x0]
-; CHECK-NEXT: splice z6.d, p0, z6.d, z7.d
-; CHECK-NEXT: st2d { z5.d, z6.d }, p1, [x0, #2, mul vl]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_factor2_wide:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v17.16b, v4.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: ptrue p1.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z5.d
+; CHECK-IAENABLED-NEXT: // kill: def $q6 killed $q6 def $z5_z6
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-IAENABLED-NEXT: st2d { z16.d, z17.d }, p1, [x0]
+; CHECK-IAENABLED-NEXT: splice z6.d, p0, z6.d, z7.d
+; CHECK-IAENABLED-NEXT: st2d { z5.d, z6.d }, p1, [x0, #2, mul vl]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_factor2_wide:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IADISABLED-NEXT: // kill: def $q6 killed $q6 def $z6
+; CHECK-IADISABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IADISABLED-NEXT: // kill: def $q4 killed $q4 def $z4
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI13_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI13_0
+; CHECK-IADISABLED-NEXT: splice z3.d, p0, z3.d, z7.d
+; CHECK-IADISABLED-NEXT: splice z2.d, p0, z2.d, z6.d
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: ldr z7, [x8]
+; CHECK-IADISABLED-NEXT: tbl z3.d, { z3.d }, z7.d
+; CHECK-IADISABLED-NEXT: tbl z2.d, { z2.d }, z7.d
+; CHECK-IADISABLED-NEXT: tbl z1.d, { z1.d }, z7.d
+; CHECK-IADISABLED-NEXT: tbl z0.d, { z0.d }, z7.d
+; CHECK-IADISABLED-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <8 x i64> %v0, <8 x i64> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -279,18 +499,38 @@ define void @load_min_not_max(ptr %ptr) #1 {
}
define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
-; CHECK-LABEL: store_min_not_max:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
-; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
-; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_min_not_max:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IAENABLED-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_min_not_max:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI17_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI17_0
+; CHECK-IADISABLED-NEXT: ptrue p1.d, vl4
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: ld1d { z2.d }, p1/z, [x8]
+; CHECK-IADISABLED-NEXT: mov x8, #4 // =0x4
+; CHECK-IADISABLED-NEXT: tbl z1.d, { z1.d }, z2.d
+; CHECK-IADISABLED-NEXT: tbl z0.d, { z0.d }, z2.d
+; CHECK-IADISABLED-NEXT: st1d { z1.d }, p1, [x0, x8, lsl #3]
+; CHECK-IADISABLED-NEXT: st1d { z0.d }, p1, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -308,18 +548,31 @@ define void @load_min_ge_type(ptr %ptr) #2 {
}
define void @store_min_ge_type(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
-; CHECK-LABEL: store_min_ge_type:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: splice z5.d, p0, z5.d, z3.d
-; CHECK-NEXT: splice z4.d, p0, z4.d, z1.d
-; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: st2d { z4.d, z5.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_min_ge_type:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z5.d, p0, z5.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z4.d, p0, z4.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IAENABLED-NEXT: st2d { z4.d, z5.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_min_ge_type:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: splice z2.d, p0, z2.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: zip1 z0.d, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i64> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -369,24 +622,49 @@ define void @load_bfloat_factor2(ptr %ptr) #0 {
}
define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
-; CHECK-LABEL: store_double_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_double_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v19.16b, v6.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-IAENABLED-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_double_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IADISABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IADISABLED-NEXT: // kill: def $q4 killed $q4 def $z4
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-IADISABLED-NEXT: // kill: def $q6 killed $q6 def $z6
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: splice z5.d, p0, z5.d, z7.d
+; CHECK-IADISABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IADISABLED-NEXT: splice z4.d, p0, z4.d, z6.d
+; CHECK-IADISABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IADISABLED-NEXT: uzp2 z2.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp1 z1.d, z1.d, z5.d
+; CHECK-IADISABLED-NEXT: uzp2 z3.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: uzp1 z0.d, z0.d, z4.d
+; CHECK-IADISABLED-NEXT: str z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z3, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -395,21 +673,92 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
}
define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8 x float> %v2) #0 {
-; CHECK-LABEL: store_float_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v17.16b, v2.16b
-; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_float_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v18.16b, v4.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.s, vl4
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v17.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-IAENABLED-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-IAENABLED-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-IAENABLED-NEXT: ptrue p0.s
+; CHECK-IAENABLED-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_float_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #144
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: add x10, sp, #96
+; CHECK-IADISABLED-NEXT: add x9, sp, #64
+; CHECK-IADISABLED-NEXT: mov w12, #20 // =0x14
+; CHECK-IADISABLED-NEXT: orr x8, x10, #0x1c
+; CHECK-IADISABLED-NEXT: st1 { v1.s }[1], [x8]
+; CHECK-IADISABLED-NEXT: orr x8, x10, #0x8
+; CHECK-IADISABLED-NEXT: st1 { v2.s }[3], [x8]
+; CHECK-IADISABLED-NEXT: orr x8, x10, #0x4
+; CHECK-IADISABLED-NEXT: st1 { v0.s }[3], [x8]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: orr x11, x8, #0xc
+; CHECK-IADISABLED-NEXT: st1 { v4.s }[2], [x8]
+; CHECK-IADISABLED-NEXT: st1 { v4.s }[3], [x11]
+; CHECK-IADISABLED-NEXT: orr x11, x9, #0x1c
+; CHECK-IADISABLED-NEXT: st1 { v2.s }[2], [x11]
+; CHECK-IADISABLED-NEXT: orr x11, x9, #0x18
+; CHECK-IADISABLED-NEXT: st1 { v0.s }[2], [x11]
+; CHECK-IADISABLED-NEXT: orr x11, x9, #0x10
+; CHECK-IADISABLED-NEXT: st1 { v2.s }[1], [x11]
+; CHECK-IADISABLED-NEXT: orr x11, x9, #0xc
+; CHECK-IADISABLED-NEXT: st1 { v0.s }[1], [x11]
+; CHECK-IADISABLED-NEXT: orr x11, x9, x12
+; CHECK-IADISABLED-NEXT: st1 { v4.s }[1], [x11]
+; CHECK-IADISABLED-NEXT: add x11, sp, #32
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0x1c
+; CHECK-IADISABLED-NEXT: orr x12, x11, x12
+; CHECK-IADISABLED-NEXT: st1 { v3.s }[1], [x11]
+; CHECK-IADISABLED-NEXT: st1 { v5.s }[3], [x13]
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0x18
+; CHECK-IADISABLED-NEXT: st1 { v3.s }[3], [x13]
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0x10
+; CHECK-IADISABLED-NEXT: st1 { v5.s }[2], [x13]
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0xc
+; CHECK-IADISABLED-NEXT: st1 { v3.s }[2], [x13]
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0x8
+; CHECK-IADISABLED-NEXT: st1 { v1.s }[2], [x13]
+; CHECK-IADISABLED-NEXT: orr x13, x11, #0x4
+; CHECK-IADISABLED-NEXT: st1 { v5.s }[1], [x13]
+; CHECK-IADISABLED-NEXT: st1 { v1.s }[3], [x12]
+; CHECK-IADISABLED-NEXT: stp s1, s3, [sp, #112]
+; CHECK-IADISABLED-NEXT: str s5, [sp, #24]
+; CHECK-IADISABLED-NEXT: ldr z1, [x10]
+; CHECK-IADISABLED-NEXT: stp s2, s4, [sp, #68]
+; CHECK-IADISABLED-NEXT: str s0, [sp, #64]
+; CHECK-IADISABLED-NEXT: mov z3.s, z1.s[7]
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[5]
+; CHECK-IADISABLED-NEXT: mov z6.s, z1.s[4]
+; CHECK-IADISABLED-NEXT: str s3, [sp, #28]
+; CHECK-IADISABLED-NEXT: mov z3.s, z1.s[2]
+; CHECK-IADISABLED-NEXT: mov z1.s, z1.s[1]
+; CHECK-IADISABLED-NEXT: stp s6, s5, [sp, #16]
+; CHECK-IADISABLED-NEXT: stp s1, s3, [sp, #4]
+; CHECK-IADISABLED-NEXT: ldr z0, [x11]
+; CHECK-IADISABLED-NEXT: ldr z1, [x9]
+; CHECK-IADISABLED-NEXT: ldr z2, [x8]
+; CHECK-IADISABLED-NEXT: str z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: str z1, [x0]
+; CHECK-IADISABLED-NEXT: str z2, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x float> %v2, <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -421,18 +770,36 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
}
define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
-; CHECK-LABEL: store_half_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov v5.16b, v2.16b
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov v4.16b, v0.16b
-; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h
-; CHECK-NEXT: splice z4.h, p0, z4.h, z1.h
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: st2h { z4.h, z5.h }, p0, [x0]
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_half_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
+; CHECK-IAENABLED-NEXT: ptrue p0.h, vl8
+; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IAENABLED-NEXT: mov v4.16b, v0.16b
+; CHECK-IAENABLED-NEXT: splice z5.h, p0, z5.h, z3.h
+; CHECK-IAENABLED-NEXT: splice z4.h, p0, z4.h, z1.h
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: st2h { z4.h, z5.h }, p0, [x0]
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_half_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ptrue p0.h, vl8
+; CHECK-IADISABLED-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-IADISABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IADISABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IADISABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IADISABLED-NEXT: adrp x8, .LCPI26_0
+; CHECK-IADISABLED-NEXT: add x8, x8, :lo12:.LCPI26_0
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-IADISABLED-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-IADISABLED-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-IADISABLED-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-IADISABLED-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: str z0, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <16 x half> %v0, <16 x half> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x half> %interleaved.vec, ptr %ptr, align 4
@@ -441,26 +808,36 @@ define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1) #0 {
-; CHECK-LABEL: store_bfloat_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: addvl x8, sp, #1
-; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: str q1, [sp, #16]
-; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: str z2, [sp, #1, mul vl]
-; CHECK-NEXT: str q3, [x8, #16]
-; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
-; CHECK-NEXT: st2h { z0.h, z1.h }, p0, [x0]
-; CHECK-NEXT: addvl sp, sp, #2
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: store_bfloat_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-IAENABLED-NEXT: addvl sp, sp, #-2
+; CHECK-IAENABLED-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-IAENABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IAENABLED-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-IAENABLED-NEXT: str z0, [sp]
+; CHECK-IAENABLED-NEXT: addvl x8, sp, #1
+; CHECK-IAENABLED-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: str q1, [sp, #16]
+; CHECK-IAENABLED-NEXT: ldr z0, [sp]
+; CHECK-IAENABLED-NEXT: str z2, [sp, #1, mul vl]
+; CHECK-IAENABLED-NEXT: str q3, [x8, #16]
+; CHECK-IAENABLED-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-IAENABLED-NEXT: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-IAENABLED-NEXT: addvl sp, sp, #2
+; CHECK-IAENABLED-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: store_bfloat_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: zip2 v4.8h, v1.8h, v3.8h
+; CHECK-IADISABLED-NEXT: zip1 v1.8h, v1.8h, v3.8h
+; CHECK-IADISABLED-NEXT: zip2 v3.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v2.8h
+; CHECK-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; CHECK-IADISABLED-NEXT: stp q0, q3, [x0]
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = shufflevector <16 x bfloat> %v0, <16 x bfloat> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
store <32 x bfloat> %interleaved.vec, ptr %ptr, align 4
>From 6cd5f7bd62efa94e2bb640d77154b6b71f3bbbdb Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Sat, 1 Aug 2026 04:51:23 +0000
Subject: [PATCH 4/5] added test with intrinsic forms
---
.../AArch64/binopshuffles-inseltpoison.ll | 159 ++-
llvm/test/CodeGen/AArch64/binopshuffles.ll | 157 +++
.../AArch64/fixed-deinterleave-intrinsics.ll | 317 ++++++
.../CodeGen/AArch64/interleaved-accesses.ll | 921 ++++++++++++++++++
.../AArch64/sve-interleaved-accesses.ll | 300 ++++++
5 files changed, 1852 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
index fff83e14ec55a..7af99acb5857b 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
@@ -171,5 +171,160 @@ entry:
%l8 = fadd fast <4 x float> %l6, %l5
ret <4 x float> %l8
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+
+define <4 x float> @twosrc_intrinsic(ptr %pSrc1, ptr %pSrc2) {
+; CHECK-IAENABLED-LABEL: twosrc_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
+; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
+ %ldN7 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN7, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN7, 1
+ %l46 = fmul fast <4 x float> %2, %1
+ %l63 = fmul fast <4 x float> %3, %0
+ %l8 = fadd fast <4 x float> %l63, %l46
+ ret <4 x float> %l8
+}
+
+define <4 x float> @twosrc2_intrinsic(ptr %pSrc1, ptr %pSrc2) {
+; CHECK-IAENABLED-LABEL: twosrc2_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc2_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
+; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
+ %ldN4 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN4, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN4, 1
+ %l43 = fmul fast <4 x float> %2, %1
+ %l6 = fmul fast <4 x float> %3, %0
+ %l8 = fadd fast <4 x float> %l6, %l43
+ ret <4 x float> %l8
+}
+
+define <4 x float> @vld2_intrinsic(ptr %pSrc) {
+; CHECK-IAENABLED-LABEL: vld2_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %l26 = fmul fast <4 x float> %2, %3
+ %l43 = fmul fast <4 x float> %0, %1
+ %l6 = fadd fast <4 x float> %l43, %l26
+ ret <4 x float> %l6
+}
+
+define <4 x float> @vld3_intrinsic(ptr %pSrc) {
+; CHECK-LABEL: vld3_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <12 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %1 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %4 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %5 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %l29 = fmul fast <4 x float> %4, %5
+ %l46 = fmul fast <4 x float> %2, %3
+ %l6 = fadd fast <4 x float> %l46, %l29
+ %l73 = fmul fast <4 x float> %0, %1
+ %l9 = fadd fast <4 x float> %l6, %l73
+ ret <4 x float> %l9
+}
+
+define <4 x float> @vld4_intrinsic(ptr %pSrc) {
+; CHECK-LABEL: vld4_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <16 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
+ %1 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %4 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %5 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %6 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %l312 = fmul fast <4 x float> %6, %7
+ %l59 = fmul fast <4 x float> %4, %5
+ %l7 = fadd fast <4 x float> %l59, %l312
+ %l86 = fmul fast <4 x float> %2, %3
+ %l103 = fmul fast <4 x float> %0, %1
+ %l12 = fadd fast <4 x float> %l103, %l86
+ ret <4 x float> %l12
+}
+
+declare { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float>)
+
+declare { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float>)
+
+declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float>)
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 66fa65ffeaecf..9535db8d7319d 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -252,3 +252,160 @@ entry:
%dead_binop = fadd <8 x double> %v0, %v0
ret void
}
+
+define <4 x float> @twosrc_intrinsic(ptr %pSrc1, ptr %pSrc2) {
+; CHECK-IAENABLED-LABEL: twosrc_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
+; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
+ %ldN7 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN7, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN7, 1
+ %l46 = fmul fast <4 x float> %2, %1
+ %l63 = fmul fast <4 x float> %3, %0
+ %l8 = fadd fast <4 x float> %l63, %l46
+ ret <4 x float> %l8
+}
+
+define <4 x float> @twosrc2_intrinsic(ptr %pSrc1, ptr %pSrc2) {
+; CHECK-IAENABLED-LABEL: twosrc2_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: twosrc2_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
+; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
+ %ldN4 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN4, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN4, 1
+ %l43 = fmul fast <4 x float> %2, %1
+ %l6 = fmul fast <4 x float> %3, %0
+ %l8 = fadd fast <4 x float> %l6, %l43
+ ret <4 x float> %l8
+}
+
+define <4 x float> @vld2_intrinsic(ptr %pSrc) {
+; CHECK-IAENABLED-LABEL: vld2_intrinsic:
+; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
+; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: vld2_intrinsic:
+; CHECK-IADISABLED: // %bb.0: // %entry
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v2.4s
+; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v1.4s
+; CHECK-IADISABLED-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <8 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
+ %2 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %3 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
+ %l26 = fmul fast <4 x float> %2, %3
+ %l43 = fmul fast <4 x float> %0, %1
+ %l6 = fadd fast <4 x float> %l43, %l26
+ ret <4 x float> %l6
+}
+
+define <4 x float> @vld3_intrinsic(ptr %pSrc) {
+; CHECK-LABEL: vld3_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <12 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %1 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %4 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %5 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %l29 = fmul fast <4 x float> %4, %5
+ %l46 = fmul fast <4 x float> %2, %3
+ %l6 = fadd fast <4 x float> %l46, %l29
+ %l73 = fmul fast <4 x float> %0, %1
+ %l9 = fadd fast <4 x float> %l6, %l73
+ ret <4 x float> %l9
+}
+
+define <4 x float> @vld4_intrinsic(ptr %pSrc) {
+; CHECK-LABEL: vld4_intrinsic:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-NEXT: ret
+entry:
+ %intrinsic.load.0 = load <16 x float>, ptr %pSrc, align 4
+ %ldN = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %intrinsic.load.0)
+ %0 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
+ %1 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
+ %2 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
+ %4 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %5 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
+ %6 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
+ %l312 = fmul fast <4 x float> %6, %7
+ %l59 = fmul fast <4 x float> %4, %5
+ %l7 = fadd fast <4 x float> %l59, %l312
+ %l86 = fmul fast <4 x float> %2, %3
+ %l103 = fmul fast <4 x float> %0, %1
+ %l12 = fadd fast <4 x float> %l103, %l86
+ ret <4 x float> %l12
+}
+
+declare { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float>)
+
+declare { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float>)
+
+declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float>)
diff --git a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
index 610e579a8f94b..f1254667d63d3 100644
--- a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
@@ -459,6 +459,323 @@ declare <8 x float> @llvm.vector.interleave2.v8f32(<4 x float>, <4 x float>)
declare <4 x double> @llvm.vector.interleave2.v4f64(<2 x double>, <2 x double>)
declare <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr>, <2 x ptr>)
declare <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr>, <8 x ptr>)
+
+define void @interleave_double_factor2_intrinsic(ptr %ptr, <2 x double> %l, <2 x double> %r) {
+; NEON-IAENABLED-LABEL: interleave_double_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_double_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_double_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <4 x double> @llvm.vector.interleave2.v4f64(<2 x double> %l, <2 x double> %r)
+ store <4 x double> %interleave, ptr %ptr, align 4
+ ret void
+}
+
+define void @interleave_float_factor2_intrinsic(ptr %ptr, <4 x float> %l, <4 x float> %r) {
+; NEON-IAENABLED-LABEL: interleave_float_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_float_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_float_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %l, <4 x float> %r)
+ store <8 x float> %interleave, ptr %ptr, align 4
+ ret void
+}
+
+define void @interleave_i16_factor2_intrinsic(ptr %ptr, <8 x i16> %l, <8 x i16> %r) {
+; NEON-IAENABLED-LABEL: interleave_i16_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.8h, v1.8h }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i16_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.8h, v0.8h, v1.8h
+; NEON-IADISABLED-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i16_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.h, z1.h[7]
+; SVE-FIXED-NEXT: mov z3.h, z0.h[7]
+; SVE-FIXED-NEXT: mov z4.h, z1.h[6]
+; SVE-FIXED-NEXT: mov z5.h, z0.h[6]
+; SVE-FIXED-NEXT: mov z6.h, z1.h[5]
+; SVE-FIXED-NEXT: mov z7.h, z0.h[5]
+; SVE-FIXED-NEXT: mov z16.h, z1.h[4]
+; SVE-FIXED-NEXT: mov z17.h, z0.h[4]
+; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z1.h
+; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
+; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %l, <8 x i16> %r)
+ store <16 x i16> %interleave, ptr %ptr, align 2
+ ret void
+}
+
+define void @interleave_i32_factor2_intrinsic(ptr %ptr, <4 x i32> %l, <4 x i32> %r) {
+; NEON-IAENABLED-LABEL: interleave_i32_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i32_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i32_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z3.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z4.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z5.s, z0.s[2]
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z1.s
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %l, <4 x i32> %r)
+ store <8 x i32> %interleave, ptr %ptr, align 4
+ ret void
+}
+
+define void @interleave_i64_factor2_intrinsic(ptr %ptr, <2 x i64> %l, <2 x i64> %r) {
+; NEON-IAENABLED-LABEL: interleave_i64_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i64_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i64_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %l, <2 x i64> %r)
+ store <4 x i64> %interleave, ptr %ptr, align 8
+ ret void
+}
+
+define void @interleave_i8_factor2_intrinsic(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
+; NEON-IAENABLED-LABEL: interleave_i8_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.16b, v1.16b }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_i8_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.16b, v0.16b, v1.16b
+; NEON-IADISABLED-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_i8_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
+; SVE-FIXED-NEXT: mov z3.b, z0.b[15]
+; SVE-FIXED-NEXT: mov z4.b, z1.b[14]
+; SVE-FIXED-NEXT: mov z5.b, z0.b[14]
+; SVE-FIXED-NEXT: mov z6.b, z1.b[13]
+; SVE-FIXED-NEXT: mov z7.b, z0.b[13]
+; SVE-FIXED-NEXT: mov z16.b, z1.b[12]
+; SVE-FIXED-NEXT: mov z17.b, z0.b[12]
+; SVE-FIXED-NEXT: mov z18.b, z1.b[11]
+; SVE-FIXED-NEXT: mov z19.b, z0.b[11]
+; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
+; SVE-FIXED-NEXT: mov z21.b, z0.b[10]
+; SVE-FIXED-NEXT: mov z22.b, z1.b[9]
+; SVE-FIXED-NEXT: mov z23.b, z0.b[9]
+; SVE-FIXED-NEXT: mov z24.b, z1.b[8]
+; SVE-FIXED-NEXT: mov z25.b, z0.b[8]
+; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
+; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
+; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
+; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
+; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
+; SVE-FIXED-NEXT: zip1 z7.b, z21.b, z20.b
+; SVE-FIXED-NEXT: zip1 z16.b, z23.b, z22.b
+; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z1.b
+; SVE-FIXED-NEXT: zip1 z17.b, z25.b, z24.b
+; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
+; SVE-FIXED-NEXT: zip1 z3.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z4.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z5.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z2.s, z3.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: stp q0, q1, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %l, <16 x i8> %r)
+ store <32 x i8> %interleave, ptr %ptr, align 1
+ ret void
+}
+
+define void @interleave_ptr_factor2_intrinsic(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
+; NEON-IAENABLED-LABEL: interleave_ptr_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_ptr_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_ptr_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: trn2 z2.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: stp q0, q2, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %l, <2 x ptr> %r)
+ store <4 x ptr> %interleave, ptr %ptr, align 4
+ ret void
+}
+
+define void @interleave_wide_ptr_factor2_intrinsic(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
+; NEON-IAENABLED-LABEL: interleave_wide_ptr_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q4_q5 def $q4_q5
+; NEON-IAENABLED-NEXT: mov v19.16b, v4.16b
+; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q6_q7 def $q6_q7
+; NEON-IAENABLED-NEXT: mov v17.16b, v6.16b
+; NEON-IAENABLED-NEXT: mov x8, x0
+; NEON-IAENABLED-NEXT: mov v18.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v6.16b, v3.16b
+; NEON-IAENABLED-NEXT: st2 { v18.2d, v19.2d }, [x8], #32
+; NEON-IAENABLED-NEXT: st2 { v4.2d, v5.2d }, [x8]
+; NEON-IAENABLED-NEXT: add x8, x0, #64
+; NEON-IAENABLED-NEXT: st2 { v16.2d, v17.2d }, [x8]
+; NEON-IAENABLED-NEXT: add x8, x0, #96
+; NEON-IAENABLED-NEXT: st2 { v6.2d, v7.2d }, [x8]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: interleave_wide_ptr_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v16.2d, v3.2d, v7.2d
+; NEON-IADISABLED-NEXT: zip1 v3.2d, v3.2d, v7.2d
+; NEON-IADISABLED-NEXT: zip2 v7.2d, v2.2d, v6.2d
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v2.2d, v6.2d
+; NEON-IADISABLED-NEXT: zip2 v6.2d, v1.2d, v5.2d
+; NEON-IADISABLED-NEXT: zip1 v1.2d, v1.2d, v5.2d
+; NEON-IADISABLED-NEXT: stp q3, q16, [x0, #96]
+; NEON-IADISABLED-NEXT: zip2 v3.2d, v0.2d, v4.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v4.2d
+; NEON-IADISABLED-NEXT: stp q1, q6, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q2, q7, [x0, #64]
+; NEON-IADISABLED-NEXT: stp q0, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; SVE-FIXED-LABEL: interleave_wide_ptr_factor2_intrinsic:
+; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-FIXED-NEXT: // kill: def $q7 killed $q7 def $z7
+; SVE-FIXED-NEXT: trn2 z16.d, z3.d, z7.d
+; SVE-FIXED-NEXT: // kill: def $q6 killed $q6 def $z6
+; SVE-FIXED-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-FIXED-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-FIXED-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-FIXED-NEXT: // kill: def $q5 killed $q5 def $z5
+; SVE-FIXED-NEXT: // kill: def $q4 killed $q4 def $z4
+; SVE-FIXED-NEXT: zip1 z3.d, z3.d, z7.d
+; SVE-FIXED-NEXT: trn2 z7.d, z2.d, z6.d
+; SVE-FIXED-NEXT: zip1 z2.d, z2.d, z6.d
+; SVE-FIXED-NEXT: trn2 z6.d, z1.d, z5.d
+; SVE-FIXED-NEXT: zip1 z1.d, z1.d, z5.d
+; SVE-FIXED-NEXT: stp q2, q7, [x0, #64]
+; SVE-FIXED-NEXT: stp q1, q6, [x0, #32]
+; SVE-FIXED-NEXT: stp q3, q16, [x0, #96]
+; SVE-FIXED-NEXT: trn2 z3.d, z0.d, z4.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z4.d
+; SVE-FIXED-NEXT: stp q0, q3, [x0]
+; SVE-FIXED-NEXT: ret
+ %interleave = tail call <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr> %l, <8 x ptr> %r)
+ store <16 x ptr> %interleave, ptr %ptr, align 4
+ ret void
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; SVE-FIXED-IADISABLED: {{.*}}
; SVE-FIXED-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index ba6699b1a5837..a706155c385a6 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -1429,6 +1429,927 @@ define <4 x i1> @load_large_vector(ptr %p) {
%ret = icmp ne <4 x ptr> %s1, %s2
ret <4 x i1> %ret
}
+
+define <4 x i1> @load_large_vector_intrinsic(ptr %p) {
+; NEON-LABEL: load_large_vector_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; NEON-NEXT: ld3 { v3.2d, v4.2d, v5.2d }, [x0]
+; NEON-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; NEON-NEXT: cmeq v1.2d, v3.2d, v4.2d
+; NEON-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; NEON-NEXT: mvn v0.16b, v0.16b
+; NEON-NEXT: xtn v0.4h, v0.4s
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: load_large_vector_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x9, x8, [x0]
+; NO_NEON-NEXT: ldp x11, x10, [x0, #24]
+; NO_NEON-NEXT: ldp x13, x12, [x0, #48]
+; NO_NEON-NEXT: cmp x9, x8
+; NO_NEON-NEXT: ldp x9, x8, [x0, #72]
+; NO_NEON-NEXT: cset w0, ne
+; NO_NEON-NEXT: cmp x11, x10
+; NO_NEON-NEXT: cset w1, ne
+; NO_NEON-NEXT: cmp x13, x12
+; NO_NEON-NEXT: cset w2, ne
+; NO_NEON-NEXT: cmp x9, x8
+; NO_NEON-NEXT: cset w3, ne
+; NO_NEON-NEXT: ret
+ %intrinsic.load.0 = load <6 x i64>, ptr %p, align 1
+ %ldN = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64> %intrinsic.load.0)
+ %1 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %ldN, 1
+ %2 = inttoptr <2 x i64> %1 to <2 x ptr>
+ %3 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %ldN, 0
+ %4 = inttoptr <2 x i64> %3 to <2 x ptr>
+ %5 = getelementptr i64, ptr %p, i32 6
+ %intrinsic.load.1 = load <6 x i64>, ptr %5, align 1
+ %ldN1 = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64> %intrinsic.load.1)
+ %6 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %ldN1, 1
+ %7 = inttoptr <2 x i64> %6 to <2 x ptr>
+ %8 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %ldN1, 0
+ %9 = inttoptr <2 x i64> %8 to <2 x ptr>
+ %10 = shufflevector <2 x ptr> %2, <2 x ptr> %7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %11 = shufflevector <2 x ptr> %4, <2 x ptr> %9, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %ret = icmp ne <4 x ptr> %11, %10
+ ret <4 x i1> %ret
+}
+
+define void @store_factor2_intrinsic(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+; NEON-IAENABLED-LABEL: store_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; NEON-IAENABLED-NEXT: // kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; NEON-IAENABLED-NEXT: st2 { v0.8b, v1.8b }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
+; NEON-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
+; NEON-IADISABLED-NEXT: adrp x8, .LCPI41_0
+; NEON-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
+; NEON-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI41_0]
+; NEON-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; NEON-IADISABLED-NEXT: str q0, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor2_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldrb w8, [sp, #64]
+; NO_NEON-NEXT: ldrb w9, [sp]
+; NO_NEON-NEXT: ldrb w10, [sp, #56]
+; NO_NEON-NEXT: strb w7, [x0, #12]
+; NO_NEON-NEXT: strb w8, [x0, #15]
+; NO_NEON-NEXT: ldrb w8, [sp, #48]
+; NO_NEON-NEXT: strb w9, [x0, #14]
+; NO_NEON-NEXT: ldrb w9, [sp, #40]
+; NO_NEON-NEXT: strb w8, [x0, #11]
+; NO_NEON-NEXT: ldrb w8, [sp, #32]
+; NO_NEON-NEXT: strb w9, [x0, #9]
+; NO_NEON-NEXT: ldrb w9, [sp, #24]
+; NO_NEON-NEXT: strb w8, [x0, #7]
+; NO_NEON-NEXT: ldrb w8, [sp, #16]
+; NO_NEON-NEXT: strb w9, [x0, #5]
+; NO_NEON-NEXT: ldrb w9, [sp, #8]
+; NO_NEON-NEXT: strb w10, [x0, #13]
+; NO_NEON-NEXT: strb w6, [x0, #10]
+; NO_NEON-NEXT: strb w5, [x0, #8]
+; NO_NEON-NEXT: strb w4, [x0, #6]
+; NO_NEON-NEXT: strb w3, [x0, #4]
+; NO_NEON-NEXT: strb w8, [x0, #3]
+; NO_NEON-NEXT: strb w2, [x0, #2]
+; NO_NEON-NEXT: strb w9, [x0, #1]
+; NO_NEON-NEXT: strb w1, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> %v0, <8 x i8> %v1)
+ store <16 x i8> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor2_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
+; NEON-IAENABLED-LABEL: store_factor2_wide_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: mov v5.16b, v2.16b
+; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
+; NEON-IAENABLED-NEXT: mov v4.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IAENABLED-NEXT: st2 { v4.4s, v5.4s }, [x0], #32
+; NEON-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor2_wide_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip2 v3.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: stp q1, q4, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q0, q3, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor2_wide_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp]
+; NO_NEON-NEXT: ldr w10, [sp, #56]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: ldr w9, [sp, #40]
+; NO_NEON-NEXT: stp w7, w10, [x0, #48]
+; NO_NEON-NEXT: stp w6, w8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: stp w5, w9, [x0, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: stp w4, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #16]
+; NO_NEON-NEXT: stp w3, w9, [x0, #16]
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: stp w2, w8, [x0, #8]
+; NO_NEON-NEXT: stp w1, w9, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
+ store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL: store_factor3_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor3_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w12, w3
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: ldr w11, [sp, #16]
+; NO_NEON-NEXT: mov w13, w6
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: // kill: def $w7 killed $w7 def $x7
+; NO_NEON-NEXT: // kill: def $w5 killed $w5 def $x5
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: orr x12, x12, x7, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x4, lsl #32
+; NO_NEON-NEXT: orr x9, x9, x2, lsl #32
+; NO_NEON-NEXT: stp x10, x8, [x0, #32]
+; NO_NEON-NEXT: mov w10, w1
+; NO_NEON-NEXT: orr x8, x13, x11, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x5, lsl #32
+; NO_NEON-NEXT: stp x8, x12, [x0, #16]
+; NO_NEON-NEXT: stp x10, x9, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
+; NEON-IAENABLED-LABEL: store_factor3_wide_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
+; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v3.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IAENABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor3_wide_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: sub sp, sp, #96
+; NEON-IADISABLED-NEXT: .cfi_def_cfa_offset 96
+; NEON-IADISABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-IADISABLED-NEXT: mov v18.16b, v4.16b
+; NEON-IADISABLED-NEXT: mov x8, sp
+; NEON-IADISABLED-NEXT: mov v17.16b, v2.16b
+; NEON-IADISABLED-NEXT: mov v4.16b, v3.16b
+; NEON-IADISABLED-NEXT: mov v16.16b, v0.16b
+; NEON-IADISABLED-NEXT: mov v3.16b, v1.16b
+; NEON-IADISABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x8]
+; NEON-IADISABLED-NEXT: add x8, sp, #48
+; NEON-IADISABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x8]
+; NEON-IADISABLED-NEXT: ldp q2, q0, [sp, #64]
+; NEON-IADISABLED-NEXT: ldp q5, q1, [sp, #32]
+; NEON-IADISABLED-NEXT: ldp q3, q4, [sp]
+; NEON-IADISABLED-NEXT: stp q2, q0, [x0, #64]
+; NEON-IADISABLED-NEXT: stp q5, q1, [x0, #32]
+; NEON-IADISABLED-NEXT: stp q3, q4, [x0]
+; NEON-IADISABLED-NEXT: add sp, sp, #96
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor3_wide_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w10, [sp, #48]
+; NO_NEON-NEXT: ldr w11, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w6 killed $w6 def $x6
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w18, w7
+; NO_NEON-NEXT: ldr w9, [sp, #64]
+; NO_NEON-NEXT: ldr w8, [sp, #120]
+; NO_NEON-NEXT: mov w17, w5
+; NO_NEON-NEXT: ldr w14, [sp, #128]
+; NO_NEON-NEXT: orr x10, x10, x11, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp]
+; NO_NEON-NEXT: ldr w12, [sp, #104]
+; NO_NEON-NEXT: ldr w13, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: ldr w15, [sp, #88]
+; NO_NEON-NEXT: orr x9, x9, x14, lsl #32
+; NO_NEON-NEXT: orr x8, x8, x11, lsl #32
+; NO_NEON-NEXT: ldr w16, [sp, #16]
+; NO_NEON-NEXT: ldr w11, [sp, #56]
+; NO_NEON-NEXT: orr x12, x12, x6, lsl #32
+; NO_NEON-NEXT: ldr w14, [sp, #40]
+; NO_NEON-NEXT: str x9, [x0, #88]
+; NO_NEON-NEXT: ldr w9, [sp, #96]
+; NO_NEON-NEXT: str x8, [x0, #80]
+; NO_NEON-NEXT: ldr w8, [sp, #80]
+; NO_NEON-NEXT: orr x11, x18, x11, lsl #32
+; NO_NEON-NEXT: orr x9, x13, x9, lsl #32
+; NO_NEON-NEXT: orr x13, x15, x4, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #56]
+; NO_NEON-NEXT: orr x8, x16, x8, lsl #32
+; NO_NEON-NEXT: ldr w12, [sp, #24]
+; NO_NEON-NEXT: str x11, [x0, #72]
+; NO_NEON-NEXT: stp x13, x9, [x0, #32]
+; NO_NEON-NEXT: ldr w13, [sp, #72]
+; NO_NEON-NEXT: mov w9, w3
+; NO_NEON-NEXT: str x8, [x0, #16]
+; NO_NEON-NEXT: ldr w8, [sp, #8]
+; NO_NEON-NEXT: mov w11, w1
+; NO_NEON-NEXT: orr x10, x17, x14, lsl #32
+; NO_NEON-NEXT: orr x9, x9, x12, lsl #32
+; NO_NEON-NEXT: orr x12, x13, x2, lsl #32
+; NO_NEON-NEXT: orr x8, x11, x8, lsl #32
+; NO_NEON-NEXT: str x10, [x0, #48]
+; NO_NEON-NEXT: str x9, [x0, #24]
+; NO_NEON-NEXT: stp x8, x12, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL: store_factor4_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor4_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: ldr w10, [sp]
+; NO_NEON-NEXT: str w7, [x0, #36]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: stp w8, w4, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w9, [x0, #40]
+; NO_NEON-NEXT: ldr w9, [sp, #16]
+; NO_NEON-NEXT: stp w8, w3, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #40]
+; NO_NEON-NEXT: str w9, [x0, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: str w6, [x0, #20]
+; NO_NEON-NEXT: stp w8, w2, [x0, #12]
+; NO_NEON-NEXT: stp w5, w9, [x0, #4]
+; NO_NEON-NEXT: str w1, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
+ store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
+; NEON-IAENABLED-LABEL: store_factor4_wide_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-IAENABLED-NEXT: mov v19.16b, v6.16b
+; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
+; NEON-IAENABLED-NEXT: mov v6.16b, v5.16b
+; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
+; NEON-IAENABLED-NEXT: mov v5.16b, v3.16b
+; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
+; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
+; NEON-IAENABLED-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_factor4_wide_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v16.4s, v3.4s, v7.4s
+; NEON-IADISABLED-NEXT: zip2 v17.4s, v1.4s, v5.4s
+; NEON-IADISABLED-NEXT: zip1 v3.4s, v3.4s, v7.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v5.4s
+; NEON-IADISABLED-NEXT: zip2 v5.4s, v2.4s, v6.4s
+; NEON-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v2.4s, v2.4s, v6.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip2 v18.4s, v17.4s, v16.4s
+; NEON-IADISABLED-NEXT: zip1 v16.4s, v17.4s, v16.4s
+; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; NEON-IADISABLED-NEXT: zip2 v3.4s, v7.4s, v5.4s
+; NEON-IADISABLED-NEXT: zip1 v5.4s, v7.4s, v5.4s
+; NEON-IADISABLED-NEXT: zip2 v6.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; NEON-IADISABLED-NEXT: stp q16, q18, [x0, #96]
+; NEON-IADISABLED-NEXT: stp q1, q4, [x0, #64]
+; NEON-IADISABLED-NEXT: stp q0, q6, [x0]
+; NEON-IADISABLED-NEXT: stp q5, q3, [x0, #32]
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor4_wide_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #192]
+; NO_NEON-NEXT: ldr w9, [sp, #128]
+; NO_NEON-NEXT: ldr w10, [sp, #64]
+; NO_NEON-NEXT: str w7, [x0, #96]
+; NO_NEON-NEXT: str w8, [x0, #124]
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: str w9, [x0, #120]
+; NO_NEON-NEXT: ldr w9, [sp, #184]
+; NO_NEON-NEXT: str w10, [x0, #116]
+; NO_NEON-NEXT: ldr w10, [sp, #120]
+; NO_NEON-NEXT: str w8, [x0, #112]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: stp w10, w9, [x0, #104]
+; NO_NEON-NEXT: ldr w9, [sp, #176]
+; NO_NEON-NEXT: ldr w10, [sp, #48]
+; NO_NEON-NEXT: str w8, [x0, #100]
+; NO_NEON-NEXT: ldr w8, [sp, #112]
+; NO_NEON-NEXT: str w10, [x0, #84]
+; NO_NEON-NEXT: ldr w10, [sp, #40]
+; NO_NEON-NEXT: stp w8, w9, [x0, #88]
+; NO_NEON-NEXT: ldr w8, [sp, #168]
+; NO_NEON-NEXT: ldr w9, [sp, #104]
+; NO_NEON-NEXT: str w10, [x0, #68]
+; NO_NEON-NEXT: ldr w10, [sp, #32]
+; NO_NEON-NEXT: stp w9, w8, [x0, #72]
+; NO_NEON-NEXT: ldr w8, [sp, #160]
+; NO_NEON-NEXT: ldr w9, [sp, #96]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #152]
+; NO_NEON-NEXT: ldr w9, [sp, #88]
+; NO_NEON-NEXT: str w10, [x0, #36]
+; NO_NEON-NEXT: ldr w10, [sp, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #144]
+; NO_NEON-NEXT: ldr w9, [sp, #80]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #8]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #136]
+; NO_NEON-NEXT: ldr w9, [sp, #72]
+; NO_NEON-NEXT: str w6, [x0, #80]
+; NO_NEON-NEXT: str w5, [x0, #64]
+; NO_NEON-NEXT: str w4, [x0, #48]
+; NO_NEON-NEXT: str w3, [x0, #32]
+; NO_NEON-NEXT: str w2, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w1, w10, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3)
+ store <32 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor3_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor3_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor3_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w10, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w6 killed $w6 def $x6
+; NO_NEON-NEXT: ldr w11, [sp, #224]
+; NO_NEON-NEXT: ldr w12, [sp, #96]
+; NO_NEON-NEXT: ldr w9, [sp, #104]
+; NO_NEON-NEXT: ldr w13, [sp, #16]
+; NO_NEON-NEXT: orr x8, x8, x10, lsl #32
+; NO_NEON-NEXT: orr x10, x11, x12, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp, #232]
+; NO_NEON-NEXT: orr x9, x9, x13, lsl #32
+; NO_NEON-NEXT: mov w13, w7
+; NO_NEON-NEXT: ldr w12, [sp, #88]
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: orr x11, x13, x11, lsl #32
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x12, x12, x6, lsl #32
+; NO_NEON-NEXT: stp x11, x9, [x0, #24]
+; NO_NEON-NEXT: mov w9, w5
+; NO_NEON-NEXT: orr x8, x9, x8, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #8]
+; NO_NEON-NEXT: str x8, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+ %intrinsic.interleave.0 = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %1, <4 x i32> %2, <4 x i32> %3)
+ store <12 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor3_undeflane_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor3_undeflane_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v0.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor3_undeflane_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: mov w14, w3
+; NO_NEON-NEXT: ldr w10, [sp, #104]
+; NO_NEON-NEXT: ldr w11, [sp, #224]
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: ldr w12, [sp, #96]
+; NO_NEON-NEXT: ldr w13, [sp, #232]
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #88]
+; NO_NEON-NEXT: orr x10, x10, x4, lsl #32
+; NO_NEON-NEXT: orr x11, x11, x12, lsl #32
+; NO_NEON-NEXT: orr x12, x14, x13, lsl #32
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x9, x9, x2, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #24]
+; NO_NEON-NEXT: mov w10, w1
+; NO_NEON-NEXT: orr x8, x10, x8, lsl #32
+; NO_NEON-NEXT: stp x9, x11, [x0, #8]
+; NO_NEON-NEXT: str x8, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+ %intrinsic.interleave.0 = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %1, <4 x i32> %2, <4 x i32> %3)
+ store <12 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor3_undefmultimid_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor3_undefmultimid_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: ldr q3, [sp]
+; NEON-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor3_undefmultimid_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #240]
+; NO_NEON-NEXT: ldr w10, [sp, #112]
+; NO_NEON-NEXT: // kill: def $w6 killed $w6 def $x6
+; NO_NEON-NEXT: ldr w11, [sp, #224]
+; NO_NEON-NEXT: ldr w12, [sp, #96]
+; NO_NEON-NEXT: ldr w9, [sp, #104]
+; NO_NEON-NEXT: ldr w13, [sp, #16]
+; NO_NEON-NEXT: orr x8, x8, x10, lsl #32
+; NO_NEON-NEXT: orr x10, x11, x12, lsl #32
+; NO_NEON-NEXT: ldr w11, [sp, #232]
+; NO_NEON-NEXT: orr x9, x9, x13, lsl #32
+; NO_NEON-NEXT: mov w13, w7
+; NO_NEON-NEXT: ldr w12, [sp, #88]
+; NO_NEON-NEXT: str x8, [x0, #40]
+; NO_NEON-NEXT: orr x11, x13, x11, lsl #32
+; NO_NEON-NEXT: ldr w8, [sp, #216]
+; NO_NEON-NEXT: orr x12, x12, x6, lsl #32
+; NO_NEON-NEXT: stp x11, x9, [x0, #24]
+; NO_NEON-NEXT: mov w9, w5
+; NO_NEON-NEXT: orr x8, x9, x8, lsl #32
+; NO_NEON-NEXT: stp x12, x10, [x0, #8]
+; NO_NEON-NEXT: str x8, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+ %intrinsic.interleave.0 = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %1, <4 x i32> %2, <4 x i32> %3)
+ store <12 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor4_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor4_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor4_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+ %4 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+ %intrinsic.interleave.0 = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %1, <2 x i32> %2, <2 x i32> %3, <2 x i32> %4)
+ store <8 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor4_undefbeg_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor4_undefbeg_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor4_undefbeg_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+ %4 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+ %intrinsic.interleave.0 = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %1, <2 x i32> %2, <2 x i32> %3, <2 x i32> %4)
+ store <8 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor4_undefend_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor4_undefend_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor4_undefend_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+ %4 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+ %intrinsic.interleave.0 = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %1, <2 x i32> %2, <2 x i32> %3, <2 x i32> %4)
+ store <8 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor4_undefmid_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor4_undefmid_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; NEON-NEXT: mov v6.16b, v2.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: ldr d5, [sp]
+; NEON-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor4_undefmid_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: ldr w9, [sp, #224]
+; NO_NEON-NEXT: ldr w10, [sp, #96]
+; NO_NEON-NEXT: str w6, [x0, #16]
+; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #216]
+; NO_NEON-NEXT: str w10, [x0, #20]
+; NO_NEON-NEXT: ldr w10, [sp, #88]
+; NO_NEON-NEXT: stp w9, w8, [x0, #8]
+; NO_NEON-NEXT: stp w5, w10, [x0]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+ %4 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+ %intrinsic.interleave.0 = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %1, <2 x i32> %2, <2 x i32> %3, <2 x i32> %4)
+ store <8 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_general_mask_factor4_undefmulti_intrinsic(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL: store_general_mask_factor4_undefmulti_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v5.16b, v2.16b
+; NEON-NEXT: mov v2.16b, v1.16b
+; NEON-NEXT: mov v3.16b, v0.16b
+; NEON-NEXT: fmov d4, d3
+; NEON-NEXT: st4 { v2.2s, v3.2s, v4.2s, v5.2s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_general_mask_factor4_undefmulti_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 16
+; NO_NEON-NEXT: .cfi_offset w29, -16
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: stp w6, w2, [x0, #16]
+; NO_NEON-NEXT: stp w5, w1, [x0]
+; NO_NEON-NEXT: stp w2, w8, [x0, #24]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: stp w1, w8, [x0, #8]
+; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; NO_NEON-NEXT: ret
+ %1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+ %2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+ %3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+ %4 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+ %intrinsic.interleave.0 = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %1, <2 x i32> %2, <2 x i32> %3, <2 x i32> %4)
+ store <8 x i32> %intrinsic.interleave.0, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
+; NEON-IAENABLED-LABEL: store_ptrvec_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_ptrvec_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_ptrvec_factor2_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: stp x3, x5, [x0, #16]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %v0, <2 x ptr> %v1)
+ store <4 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2) {
+; NEON-LABEL: store_ptrvec_factor3_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_ptrvec_factor3_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: stp x5, x7, [x0, #32]
+; NO_NEON-NEXT: stp x6, x3, [x0, #16]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <6 x ptr> @llvm.vector.interleave3.v6p0(<2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2)
+ store <6 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2, <2 x ptr> %v3) {
+; NEON-LABEL: store_ptrvec_factor4_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_ptrvec_factor4_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr x8, [sp, #8]
+; NO_NEON-NEXT: stp x3, x5, [x0, #32]
+; NO_NEON-NEXT: stp x2, x4, [x0]
+; NO_NEON-NEXT: stp x7, x8, [x0, #48]
+; NO_NEON-NEXT: ldr x8, [sp]
+; NO_NEON-NEXT: stp x6, x8, [x0, #16]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <8 x ptr> @llvm.vector.interleave4.v8p0(<2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2, <2 x ptr> %v3)
+ store <8 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_undef_mask_factor2_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+; NEON-IAENABLED-LABEL: store_undef_mask_factor2_intrinsic:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; NEON-IAENABLED-NEXT: st2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: store_undef_mask_factor2_intrinsic:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; NEON-IADISABLED-NEXT: stp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ret
+;
+; NO_NEON-LABEL: store_undef_mask_factor2_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: stp w3, w7, [x0, #16]
+; NO_NEON-NEXT: stp w2, w6, [x0, #8]
+; NO_NEON-NEXT: stp w4, w8, [x0, #24]
+; NO_NEON-NEXT: stp w1, w5, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_undef_mask_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL: store_undef_mask_factor3_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; NEON-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_undef_mask_factor3_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: // kill: def $w4 killed $w4 def $x4
+; NO_NEON-NEXT: mov w12, w3
+; NO_NEON-NEXT: ldr w10, [sp, #24]
+; NO_NEON-NEXT: ldr w11, [sp, #16]
+; NO_NEON-NEXT: mov w13, w6
+; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: // kill: def $w7 killed $w7 def $x7
+; NO_NEON-NEXT: // kill: def $w5 killed $w5 def $x5
+; NO_NEON-NEXT: // kill: def $w2 killed $w2 def $x2
+; NO_NEON-NEXT: orr x12, x12, x7, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x4, lsl #32
+; NO_NEON-NEXT: orr x9, x9, x2, lsl #32
+; NO_NEON-NEXT: stp x10, x8, [x0, #32]
+; NO_NEON-NEXT: mov w10, w1
+; NO_NEON-NEXT: orr x8, x13, x11, lsl #32
+; NO_NEON-NEXT: orr x10, x10, x5, lsl #32
+; NO_NEON-NEXT: stp x8, x12, [x0, #16]
+; NO_NEON-NEXT: stp x10, x9, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_undef_mask_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL: store_undef_mask_factor4_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; NEON-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_undef_mask_factor4_intrinsic:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr w8, [sp, #64]
+; NO_NEON-NEXT: ldr w9, [sp, #32]
+; NO_NEON-NEXT: ldr w10, [sp]
+; NO_NEON-NEXT: str w7, [x0, #36]
+; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldr w8, [sp, #56]
+; NO_NEON-NEXT: ldr w9, [sp, #24]
+; NO_NEON-NEXT: str w10, [x0, #52]
+; NO_NEON-NEXT: stp w8, w4, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w9, [x0, #40]
+; NO_NEON-NEXT: ldr w9, [sp, #16]
+; NO_NEON-NEXT: stp w8, w3, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #40]
+; NO_NEON-NEXT: str w9, [x0, #24]
+; NO_NEON-NEXT: ldr w9, [sp, #8]
+; NO_NEON-NEXT: str w6, [x0, #20]
+; NO_NEON-NEXT: stp w8, w2, [x0, #12]
+; NO_NEON-NEXT: stp w5, w9, [x0, #4]
+; NO_NEON-NEXT: str w1, [x0]
+; NO_NEON-NEXT: ret
+ %interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
+ store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
+ ret void
+}
+
+declare <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32>, <4 x i32>, <4 x i32>)
+
+declare <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32>, <8 x i32>)
+
+declare <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>)
+
+declare <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8>, <8 x i8>)
+
+declare <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32>, <8 x i32>, <8 x i32>)
+
+declare <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32>, <8 x i32>, <8 x i32>, <8 x i32>)
+
+declare <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr>, <2 x ptr>)
+
+declare <6 x ptr> @llvm.vector.interleave3.v6p0(<2 x ptr>, <2 x ptr>, <2 x ptr>)
+
+declare <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32>, <4 x i32>)
+
+declare <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>)
+
+declare <8 x ptr> @llvm.vector.interleave4.v8p0(<2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr>)
+
+declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64>)
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; NO_NEON-IADISABLED: {{.*}}
; NO_NEON-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index aa9684bef45c1..ff10464b1de89 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -856,6 +856,306 @@ define void @deinterleave_nxptr_factor2(ptr %ptr) #2 {
ret void
}
+define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
+; CHECK-LABEL: store_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI29_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI29_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
+ store <32 x i16> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-LABEL: store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-LABEL: store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
+; CHECK-LABEL: store_ptrvec_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI32_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI32_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-NEXT: tbl z1.d, { z1.d }, z4.d
+; CHECK-NEXT: tbl z0.d, { z0.d }, z4.d
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <8 x ptr> @llvm.vector.interleave2.v8p0(<4 x ptr> %v0, <4 x ptr> %v1)
+ store <8 x ptr> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2) #0 {
+; CHECK-LABEL: store_ptrvec_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st3d { z16.d - z18.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <12 x ptr> @llvm.vector.interleave3.v12p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2)
+ store <12 x ptr> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
+; CHECK-LABEL: store_ptrvec_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x ptr> @llvm.vector.interleave4.v16p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3)
+ store <16 x ptr> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor2_wide_intrinsic(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
+; CHECK-LABEL: store_factor2_wide_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q6 killed $q6 def $z6
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q4 killed $q4 def $z4
+; CHECK-NEXT: adrp x8, .LCPI35_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI35_0
+; CHECK-NEXT: splice z3.d, p0, z3.d, z7.d
+; CHECK-NEXT: splice z2.d, p0, z2.d, z6.d
+; CHECK-NEXT: splice z1.d, p0, z1.d, z5.d
+; CHECK-NEXT: splice z0.d, p0, z0.d, z4.d
+; CHECK-NEXT: ldr z7, [x8]
+; CHECK-NEXT: tbl z3.d, { z3.d }, z7.d
+; CHECK-NEXT: tbl z2.d, { z2.d }, z7.d
+; CHECK-NEXT: tbl z1.d, { z1.d }, z7.d
+; CHECK-NEXT: tbl z0.d, { z0.d }, z7.d
+; CHECK-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave2.v16i64(<8 x i64> %v0, <8 x i64> %v1)
+ store <16 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_min_not_max_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
+; CHECK-LABEL: store_min_not_max_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI36_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI36_0
+; CHECK-NEXT: ptrue p1.d, vl4
+; CHECK-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x8]
+; CHECK-NEXT: mov x8, #4 // =0x4
+; CHECK-NEXT: tbl z1.d, { z1.d }, z2.d
+; CHECK-NEXT: tbl z0.d, { z0.d }, z2.d
+; CHECK-NEXT: st1d { z1.d }, p1, [x0, x8, lsl #3]
+; CHECK-NEXT: st1d { z0.d }, p1, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> %v0, <4 x i64> %v1)
+ store <8 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_min_ge_type_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
+; CHECK-LABEL: store_min_ge_type_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: adrp x8, .LCPI37_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI37_0
+; CHECK-NEXT: splice z2.d, p0, z2.d, z3.d
+; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ldr z1, [x8]
+; CHECK-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-NEXT: tbl z0.d, { z0.d }, z1.d
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <8 x i64> @llvm.vector.interleave2.v8i64(<4 x i64> %v0, <4 x i64> %v1)
+ store <8 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_double_factor4_intrinsic(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
+; CHECK-LABEL: store_double_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x double> @llvm.vector.interleave4.v16f64(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3)
+ store <16 x double> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_float_factor3_intrinsic(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8 x float> %v2) #0 {
+; CHECK-LABEL: store_float_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x float> @llvm.vector.interleave3.v24f32(<8 x float> %v0, <8 x float> %v1, <8 x float> %v2)
+ store <24 x float> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_half_factor2_intrinsic(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
+; CHECK-LABEL: store_half_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI40_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI40_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x half> @llvm.vector.interleave2.v32f16(<16 x half> %v0, <16 x half> %v1)
+ store <32 x half> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_bfloat_factor2_intrinsic(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1) #0 {
+; CHECK-LABEL: store_bfloat_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: zip2 v4.8h, v1.8h, v3.8h
+; CHECK-NEXT: zip1 v1.8h, v1.8h, v3.8h
+; CHECK-NEXT: zip2 v3.8h, v0.8h, v2.8h
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT: stp q1, q4, [x0, #32]
+; CHECK-NEXT: stp q0, q3, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x bfloat> @llvm.vector.interleave2.v32bf16(<16 x bfloat> %v0, <16 x bfloat> %v1)
+ store <32 x bfloat> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
attributes #1 = { vscale_range(2,4) "target-features"="+sve" }
attributes #2 = { vscale_range(4,4) "target-features"="+sve" }
>From fa3f2ff13b1391e128662ec1f49664df117f4876 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Mon, 10 Aug 2026 08:49:54 +0000
Subject: [PATCH 5/5] fixup keep testcases by marking fake use
---
.../AArch64/binopshuffles-inseltpoison.ll | 330 ------
llvm/test/CodeGen/AArch64/binopshuffles.ll | 70 +-
.../AArch64/fixed-deinterleave-intrinsics.ll | 370 +++++-
...aved-accesses-extract-user-inseltpoison.ll | 95 --
.../interleaved-accesses-extract-user.ll | 95 --
.../CodeGen/AArch64/interleaved-accesses.ll | 1018 ++++++++++++++---
.../AArch64/sve-interleaved-accesses.ll | 931 +++++++++++++--
7 files changed, 2053 insertions(+), 856 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
delete mode 100644 llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
delete mode 100644 llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
deleted file mode 100644
index 7af99acb5857b..0000000000000
--- a/llvm/test/CodeGen/AArch64/binopshuffles-inseltpoison.ll
+++ /dev/null
@@ -1,330 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
-
-define <4 x float> @vld2(ptr %pSrc) {
-; CHECK-IAENABLED-LABEL: vld2:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld2:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %wide.vec = load <8 x float>, ptr %pSrc, align 4
- %l2 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l3 = shufflevector <8 x float> %l2, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %l4 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
- %l6 = fadd fast <4 x float> %l5, %l3
- ret <4 x float> %l6
-}
-
-define <4 x float> @vld3(ptr %pSrc) {
-; CHECK-IAENABLED-LABEL: vld3:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v3.4s, v3.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld3:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: ldr q4, [x0, #32]
-; CHECK-IADISABLED-NEXT: fmul v4.4s, v4.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: mov v2.16b, v0.16b
-; CHECK-IADISABLED-NEXT: rev64 v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
-; CHECK-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
-; CHECK-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
-; CHECK-IADISABLED-NEXT: mov v1.s[0], v0.s[2]
-; CHECK-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
-; CHECK-IADISABLED-NEXT: mov v2.s[3], v4.s[1]
-; CHECK-IADISABLED-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-IADISABLED-NEXT: fadd v0.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
-; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %wide.vec = load <12 x float>, ptr %pSrc, align 4
- %l2 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l3 = shufflevector <12 x float> %l2, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
- %l4 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l5 = shufflevector <12 x float> %l4, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %l6 = fadd fast <4 x float> %l5, %l3
- %l7 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l8 = shufflevector <12 x float> %l7, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
- %l9 = fadd fast <4 x float> %l6, %l8
- ret <4 x float> %l9
-}
-
-define <4 x float> @vld4(ptr %pSrc) {
-; CHECK-IAENABLED-LABEL: vld4:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v3.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v4.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld4:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: ldp q2, q3, [x0, #32]
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v3.4s, v3.4s, v3.4s
-; CHECK-IADISABLED-NEXT: fmul v2.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v4.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: zip2 v1.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: trn1 v5.4s, v2.4s, v3.4s
-; CHECK-IADISABLED-NEXT: zip2 v2.4s, v2.4s, v3.4s
-; CHECK-IADISABLED-NEXT: uzp2 v0.4s, v4.4s, v0.4s
-; CHECK-IADISABLED-NEXT: mov v1.d[1], v5.d[1]
-; CHECK-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
-; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %wide.vec = load <16 x float>, ptr %pSrc, align 4
- %l3 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l4 = shufflevector <16 x float> %l3, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
- %l5 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l6 = shufflevector <16 x float> %l5, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
- %l7 = fadd fast <4 x float> %l6, %l4
- %l8 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l9 = shufflevector <16 x float> %l8, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
- %l10 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l11 = shufflevector <16 x float> %l10, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
- %l12 = fadd fast <4 x float> %l11, %l9
- ret <4 x float> %l12
-}
-
-define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q0, q1, [x0]
-; CHECK-IADISABLED-NEXT: ldp q2, q3, [x1]
-; CHECK-IADISABLED-NEXT: fmul v1.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v0.4s
-; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %wide.vec = load <8 x float>, ptr %pSrc1, align 4
- %wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
- %l4 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %l6 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l7 = shufflevector <8 x float> %l6, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
- %l8 = fadd fast <4 x float> %l7, %l5
- ret <4 x float> %l8
-}
-
-define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc2:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc2:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q2, [x0]
-; CHECK-IADISABLED-NEXT: ldp q3, q4, [x1]
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v4.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmul v5.4s, v3.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp2 v3.4s, v3.4s, v4.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v5.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v3.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %wide.vec = load <8 x float>, ptr %pSrc1, align 4
- %wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
- %l4 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %s1 = shufflevector <8 x float> %wide.vec26, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
- %s2 = shufflevector <8 x float> %wide.vec, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
- %l6 = fmul fast <4 x float> %s1, %s2
- %l8 = fadd fast <4 x float> %l6, %l5
- ret <4 x float> %l8
-}
-
-define <4 x float> @twosrc_intrinsic(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
-; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
- %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
- %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
- %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
- %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
- %ldN7 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
- %2 = extractvalue { <4 x float>, <4 x float> } %ldN7, 0
- %3 = extractvalue { <4 x float>, <4 x float> } %ldN7, 1
- %l46 = fmul fast <4 x float> %2, %1
- %l63 = fmul fast <4 x float> %3, %0
- %l8 = fadd fast <4 x float> %l63, %l46
- ret <4 x float> %l8
-}
-
-define <4 x float> @twosrc2_intrinsic(ptr %pSrc1, ptr %pSrc2) {
-; CHECK-IAENABLED-LABEL: twosrc2_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: ld2 { v3.4s, v4.4s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v3.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v4.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: twosrc2_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: ldp q3, q2, [x1]
-; CHECK-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
- %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
- %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
- %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
- %intrinsic.load.1 = load <8 x float>, ptr %pSrc2, align 4
- %ldN4 = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.1)
- %2 = extractvalue { <4 x float>, <4 x float> } %ldN4, 0
- %3 = extractvalue { <4 x float>, <4 x float> } %ldN4, 1
- %l43 = fmul fast <4 x float> %2, %1
- %l6 = fmul fast <4 x float> %3, %0
- %l8 = fadd fast <4 x float> %l6, %l43
- ret <4 x float> %l8
-}
-
-define <4 x float> @vld2_intrinsic(ptr %pSrc) {
-; CHECK-IAENABLED-LABEL: vld2_intrinsic:
-; CHECK-IAENABLED: // %bb.0: // %entry
-; CHECK-IAENABLED-NEXT: ld2 { v1.4s, v2.4s }, [x0]
-; CHECK-IAENABLED-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-IAENABLED-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-IAENABLED-NEXT: ret
-;
-; CHECK-IADISABLED-LABEL: vld2_intrinsic:
-; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: uzp2 v1.4s, v1.4s, v0.4s
-; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v2.4s
-; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v1.4s
-; CHECK-IADISABLED-NEXT: ret
-entry:
- %intrinsic.load.0 = load <8 x float>, ptr %pSrc, align 4
- %ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
- %0 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
- %1 = extractvalue { <4 x float>, <4 x float> } %ldN, 1
- %2 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
- %3 = extractvalue { <4 x float>, <4 x float> } %ldN, 0
- %l26 = fmul fast <4 x float> %2, %3
- %l43 = fmul fast <4 x float> %0, %1
- %l6 = fadd fast <4 x float> %l43, %l26
- ret <4 x float> %l6
-}
-
-define <4 x float> @vld3_intrinsic(ptr %pSrc) {
-; CHECK-LABEL: vld3_intrinsic:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v1.4s, v1.4s
-; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
-; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: ret
-entry:
- %intrinsic.load.0 = load <12 x float>, ptr %pSrc, align 4
- %ldN = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %intrinsic.load.0)
- %0 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
- %1 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 2
- %2 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
- %3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 1
- %4 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
- %5 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %ldN, 0
- %l29 = fmul fast <4 x float> %4, %5
- %l46 = fmul fast <4 x float> %2, %3
- %l6 = fadd fast <4 x float> %l46, %l29
- %l73 = fmul fast <4 x float> %0, %1
- %l9 = fadd fast <4 x float> %l6, %l73
- ret <4 x float> %l9
-}
-
-define <4 x float> @vld4_intrinsic(ptr %pSrc) {
-; CHECK-LABEL: vld4_intrinsic:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
-; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
-; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
-; CHECK-NEXT: ret
-entry:
- %intrinsic.load.0 = load <16 x float>, ptr %pSrc, align 4
- %ldN = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %intrinsic.load.0)
- %0 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
- %1 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 3
- %2 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
- %3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 2
- %4 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
- %5 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 1
- %6 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
- %7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %ldN, 0
- %l312 = fmul fast <4 x float> %6, %7
- %l59 = fmul fast <4 x float> %4, %5
- %l7 = fadd fast <4 x float> %l59, %l312
- %l86 = fmul fast <4 x float> %2, %3
- %l103 = fmul fast <4 x float> %0, %1
- %l12 = fadd fast <4 x float> %l103, %l86
- ret <4 x float> %l12
-}
-
-declare { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float>)
-
-declare { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float>)
-
-declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float>)
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 9535db8d7319d..0dd129b403b89 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -1,9 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
define <4 x float> @vld2(ptr %pSrc) {
; CHECK-IAENABLED-LABEL: vld2:
@@ -23,9 +20,9 @@ define <4 x float> @vld2(ptr %pSrc) {
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l3 = shufflevector <8 x float> %l2, <8 x float> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %l3 = shufflevector <8 x float> %l2, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%l4 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
%l6 = fadd fast <4 x float> %l5, %l3
ret <4 x float> %l6
}
@@ -62,12 +59,12 @@ define <4 x float> @vld3(ptr %pSrc) {
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l3 = shufflevector <12 x float> %l2, <12 x float> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+ %l3 = shufflevector <12 x float> %l2, <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%l4 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l5 = shufflevector <12 x float> %l4, <12 x float> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+ %l5 = shufflevector <12 x float> %l4, <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
%l6 = fadd fast <4 x float> %l5, %l3
%l7 = fmul fast <12 x float> %wide.vec, %wide.vec
- %l8 = shufflevector <12 x float> %l7, <12 x float> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ %l8 = shufflevector <12 x float> %l7, <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
%l9 = fadd fast <4 x float> %l6, %l8
ret <4 x float> %l9
}
@@ -100,14 +97,14 @@ define <4 x float> @vld4(ptr %pSrc) {
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l4 = shufflevector <16 x float> %l3, <16 x float> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+ %l4 = shufflevector <16 x float> %l3, <16 x float> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%l5 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l6 = shufflevector <16 x float> %l5, <16 x float> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+ %l6 = shufflevector <16 x float> %l5, <16 x float> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%l7 = fadd fast <4 x float> %l6, %l4
%l8 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l9 = shufflevector <16 x float> %l8, <16 x float> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+ %l9 = shufflevector <16 x float> %l8, <16 x float> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%l10 = fmul fast <16 x float> %wide.vec, %wide.vec
- %l11 = shufflevector <16 x float> %l10, <16 x float> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ %l11 = shufflevector <16 x float> %l10, <16 x float> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
%l12 = fadd fast <4 x float> %l11, %l9
ret <4 x float> %l12
}
@@ -133,9 +130,9 @@ entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
%l4 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%l6 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l7 = shufflevector <8 x float> %l6, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %l7 = shufflevector <8 x float> %l6, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
%l8 = fadd fast <4 x float> %l7, %l5
ret <4 x float> %l8
}
@@ -164,9 +161,9 @@ entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
%l4 = fmul fast <8 x float> %wide.vec26, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %s1 = shufflevector <8 x float> %wide.vec26, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
- %s2 = shufflevector <8 x float> %wide.vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %s1 = shufflevector <8 x float> %wide.vec26, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %s2 = shufflevector <8 x float> %wide.vec, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
%l6 = fmul fast <4 x float> %s1, %s2
%l8 = fadd fast <4 x float> %l6, %l5
ret <4 x float> %l8
@@ -188,19 +185,6 @@ entry:
ret void
}
-define void @noncanonical2(ptr %p0, ptr %p1, ptr %p2) {
-; CHECK-LABEL: noncanonical2:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %v0 = load <8 x i8>, ptr %p0
- %v1 = load <8 x i8>, ptr %p1
- %v2 = add <8 x i8> %v0, %v1
- %shuffled = shufflevector <8 x i8> undef, <8 x i8> %v2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- store <4 x i8> %shuffled, ptr %p2
- ret void
-}
-
define <4 x float> @noncanonical3(ptr %pSrc) {
; CHECK-LABEL: noncanonical3:
; CHECK: // %bb.0: // %entry
@@ -212,9 +196,9 @@ define <4 x float> @noncanonical3(ptr %pSrc) {
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l3 = shufflevector <8 x float> undef, <8 x float> %l2, <4 x i32> <i32 8, i32 10, i32 12, i32 14>
+ %l3 = shufflevector <8 x float> poison, <8 x float> %l2, <4 x i32> <i32 8, i32 10, i32 12, i32 14>
%l4 = fmul fast <8 x float> %wide.vec, %wide.vec
- %l5 = shufflevector <8 x float> %l4, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %l5 = shufflevector <8 x float> %l4, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
%l6 = fadd fast <4 x float> %l5, %l3
ret <4 x float> %l6
}
@@ -222,9 +206,9 @@ entry:
define void @noncanonical_extmask(ptr %p0, ptr %p1, ptr %p2) {
; CHECK-LABEL: noncanonical_extmask:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: adrp x8, .LCPI8_0
+; CHECK-NEXT: adrp x8, .LCPI7_0
; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI8_0]
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI7_0]
; CHECK-NEXT: add v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ldr d1, [x1]
; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b
@@ -245,10 +229,10 @@ define void @skip_optimizing_dead_binop(ptr %p0, ptr %p1) {
; CHECK-NEXT: ret
entry:
%v0 = load <8 x double>, ptr %p0
- %shuffled_1 = shufflevector <8 x double> %v0, <8 x double> undef, <2 x i32> <i32 0, i32 4>
- %shuffled_2 = shufflevector <8 x double> %v0, <8 x double> undef, <2 x i32> <i32 1, i32 5>
- %shuffled_3 = shufflevector <8 x double> %v0, <8 x double> undef, <2 x i32> <i32 2, i32 6>
- %shuffled_4 = shufflevector <8 x double> %v0, <8 x double> undef, <2 x i32> <i32 3, i32 7>
+ %shuffled_1 = shufflevector <8 x double> %v0, <8 x double> poison, <2 x i32> <i32 0, i32 4>
+ %shuffled_2 = shufflevector <8 x double> %v0, <8 x double> poison, <2 x i32> <i32 1, i32 5>
+ %shuffled_3 = shufflevector <8 x double> %v0, <8 x double> poison, <2 x i32> <i32 2, i32 6>
+ %shuffled_4 = shufflevector <8 x double> %v0, <8 x double> poison, <2 x i32> <i32 3, i32 7>
%dead_binop = fadd <8 x double> %v0, %v0
ret void
}
@@ -403,9 +387,3 @@ entry:
%l12 = fadd fast <4 x float> %l103, %l86
ret <4 x float> %l12
}
-
-declare { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float>)
-
-declare { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float>)
-
-declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float>)
diff --git a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
index f1254667d63d3..91db2ce7d589b 100644
--- a/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-deinterleave-intrinsics.ll
@@ -1,113 +1,292 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
-; RUN: llc < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IADISABLED
-
-target triple = "aarch64-linux-gnu"
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s -mattr=+sve -force-streaming-compatible | FileCheck %s --check-prefixes=SVE-FIXED,SVE-FIXED-IADISABLED
define void @deinterleave_i8_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_i8_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_i8_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.16b, v1.16b }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_i8_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: uzp1 v2.16b, v1.16b, v0.16b
+; NEON-IADISABLED-NEXT: uzp2 v0.16b, v1.16b, v0.16b
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_i8_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
+; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE-FIXED-NEXT: .cfi_offset b8, -16
+; SVE-FIXED-NEXT: ldp q0, q1, [x0]
+; SVE-FIXED-NEXT: mov z2.b, z1.b[15]
+; SVE-FIXED-NEXT: mov z3.b, z1.b[13]
+; SVE-FIXED-NEXT: mov z4.b, z1.b[11]
+; SVE-FIXED-NEXT: mov z5.b, z1.b[9]
+; SVE-FIXED-NEXT: mov z6.b, z1.b[7]
+; SVE-FIXED-NEXT: mov z7.b, z1.b[5]
+; SVE-FIXED-NEXT: mov z18.b, z0.b[15]
+; SVE-FIXED-NEXT: mov z19.b, z0.b[13]
+; SVE-FIXED-NEXT: mov z20.b, z1.b[14]
+; SVE-FIXED-NEXT: mov z21.b, z1.b[12]
+; SVE-FIXED-NEXT: mov z16.b, z1.b[3]
+; SVE-FIXED-NEXT: mov z17.b, z1.b[1]
+; SVE-FIXED-NEXT: zip1 z2.b, z3.b, z2.b
+; SVE-FIXED-NEXT: zip1 z3.b, z5.b, z4.b
+; SVE-FIXED-NEXT: zip1 z4.b, z7.b, z6.b
+; SVE-FIXED-NEXT: zip1 z6.b, z19.b, z18.b
+; SVE-FIXED-NEXT: mov z22.b, z1.b[6]
+; SVE-FIXED-NEXT: mov z23.b, z1.b[4]
+; SVE-FIXED-NEXT: zip1 z19.b, z21.b, z20.b
+; SVE-FIXED-NEXT: mov z20.b, z1.b[10]
+; SVE-FIXED-NEXT: mov z21.b, z1.b[8]
+; SVE-FIXED-NEXT: mov z24.b, z1.b[2]
+; SVE-FIXED-NEXT: mov z25.b, z0.b[14]
+; SVE-FIXED-NEXT: mov z26.b, z0.b[12]
+; SVE-FIXED-NEXT: mov z27.b, z0.b[10]
+; SVE-FIXED-NEXT: mov z28.b, z0.b[8]
+; SVE-FIXED-NEXT: mov z29.b, z0.b[6]
+; SVE-FIXED-NEXT: mov z30.b, z0.b[4]
+; SVE-FIXED-NEXT: mov z31.b, z0.b[2]
+; SVE-FIXED-NEXT: zip1 z5.b, z17.b, z16.b
+; SVE-FIXED-NEXT: mov z7.b, z0.b[11]
+; SVE-FIXED-NEXT: mov z16.b, z0.b[9]
+; SVE-FIXED-NEXT: mov z17.b, z0.b[7]
+; SVE-FIXED-NEXT: mov z18.b, z0.b[5]
+; SVE-FIXED-NEXT: mov z8.b, z0.b[3]
+; SVE-FIXED-NEXT: zip1 z20.b, z21.b, z20.b
+; SVE-FIXED-NEXT: mov z21.b, z0.b[1]
+; SVE-FIXED-NEXT: zip1 z22.b, z23.b, z22.b
+; SVE-FIXED-NEXT: zip1 z1.b, z1.b, z24.b
+; SVE-FIXED-NEXT: zip1 z23.b, z26.b, z25.b
+; SVE-FIXED-NEXT: zip1 z24.b, z28.b, z27.b
+; SVE-FIXED-NEXT: zip1 z25.b, z30.b, z29.b
+; SVE-FIXED-NEXT: zip1 z0.b, z0.b, z31.b
+; SVE-FIXED-NEXT: zip1 z7.b, z16.b, z7.b
+; SVE-FIXED-NEXT: zip1 z16.b, z18.b, z17.b
+; SVE-FIXED-NEXT: zip1 z17.b, z21.b, z8.b
+; SVE-FIXED-NEXT: zip1 z2.h, z3.h, z2.h
+; SVE-FIXED-NEXT: zip1 z3.h, z20.h, z19.h
+; SVE-FIXED-NEXT: zip1 z1.h, z1.h, z22.h
+; SVE-FIXED-NEXT: zip1 z18.h, z24.h, z23.h
+; SVE-FIXED-NEXT: zip1 z4.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z25.h
+; SVE-FIXED-NEXT: zip1 z5.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z6.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
+; SVE-FIXED-NEXT: zip1 z2.s, z4.s, z2.s
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z18.s
+; SVE-FIXED-NEXT: zip1 z3.s, z6.s, z5.s
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: // fake_use: $q1 $z1
+; SVE-FIXED-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload
; SVE-FIXED-NEXT: ret
%load = load <32 x i8>, ptr %ptr, align 1
%deinterleave = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %load)
%extract1 = extractvalue { <16 x i8>, <16 x i8> } %deinterleave, 0
%extract2 = extractvalue { <16 x i8>, <16 x i8> } %deinterleave, 1
- ret void
-}
-
-define void @deinterleave_i16_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_i16_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
-;
-; SVE-FIXED-LABEL: deinterleave_i16_factor2:
-; SVE-FIXED: // %bb.0:
-; SVE-FIXED-NEXT: ret
- %load = load <16 x i16>, ptr %ptr, align 2
- %deinterleave = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %load)
- %extract1 = extractvalue { <8 x i16>, <8 x i16> } %deinterleave, 0
- %extract2 = extractvalue { <8 x i16>, <8 x i16> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<16 x i8> %extract1)
+ tail call void (...) @llvm.fake.use(<16 x i8> %extract2)
ret void
}
define void @deinterleave_8xi32_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_8xi32_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_8xi32_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_8xi32_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_8xi32_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ldp q0, q1, [x0]
+; SVE-FIXED-NEXT: mov z3.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z4.s, z0.s[2]
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z5.s, z1.s[1]
+; SVE-FIXED-NEXT: mov z6.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z7.s, z0.s[1]
+; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z4.s
+; SVE-FIXED-NEXT: zip1 z2.s, z5.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z7.s, z6.s
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: // fake_use: $q1 $z1
; SVE-FIXED-NEXT: ret
%load = load <8 x i32>, ptr %ptr, align 4
%deinterleave = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %load)
%extract1 = extractvalue { <4 x i32>, <4 x i32> } %deinterleave, 0
%extract2 = extractvalue { <4 x i32>, <4 x i32> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<4 x i32> %extract1)
+ tail call void (...) @llvm.fake.use(<4 x i32> %extract2)
ret void
}
define void @deinterleave_i64_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_i64_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_i64_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_i64_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_i64_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ldp q1, q0, [x0]
+; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
+; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
+; SVE-FIXED-NEXT: // fake_use: $q2 $z2
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
; SVE-FIXED-NEXT: ret
%load = load <4 x i64>, ptr %ptr, align 8
%deinterleave = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %load)
%extract1 = extractvalue { <2 x i64>, <2 x i64> } %deinterleave, 0
%extract2 = extractvalue { <2 x i64>, <2 x i64> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<2 x i64> %extract1)
+ tail call void (...) @llvm.fake.use(<2 x i64> %extract2)
ret void
}
define void @deinterleave_float_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_float_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_float_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_float_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_float_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ldp q0, q1, [x0]
+; SVE-FIXED-NEXT: mov z3.s, z1.s[2]
+; SVE-FIXED-NEXT: mov z4.s, z0.s[2]
+; SVE-FIXED-NEXT: mov z2.s, z1.s[3]
+; SVE-FIXED-NEXT: mov z5.s, z1.s[1]
+; SVE-FIXED-NEXT: mov z6.s, z0.s[3]
+; SVE-FIXED-NEXT: mov z7.s, z0.s[1]
+; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z3.s
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z4.s
+; SVE-FIXED-NEXT: zip1 z2.s, z5.s, z2.s
+; SVE-FIXED-NEXT: zip1 z3.s, z7.s, z6.s
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: zip1 z1.d, z3.d, z2.d
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: // fake_use: $q1 $z1
; SVE-FIXED-NEXT: ret
%load = load <8 x float>, ptr %ptr, align 4
%deinterleave = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %load)
%extract1 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
%extract2 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<4 x float> %extract1)
+ tail call void (...) @llvm.fake.use(<4 x float> %extract2)
ret void
}
define void @deinterleave_double_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_double_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_double_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_double_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_double_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ldp q1, q0, [x0]
+; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
+; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
+; SVE-FIXED-NEXT: // fake_use: $q2 $z2
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
; SVE-FIXED-NEXT: ret
%load = load <4 x double>, ptr %ptr, align 8
%deinterleave = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %load)
%extract1 = extractvalue { <2 x double>, <2 x double> } %deinterleave, 0
%extract2 = extractvalue { <2 x double>, <2 x double> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<2 x double> %extract1)
+ tail call void (...) @llvm.fake.use(<2 x double> %extract2)
ret void
}
define void @deinterleave_ptr_factor2(ptr %ptr) {
-; NEON-LABEL: deinterleave_ptr_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_ptr_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_ptr_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_ptr_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: ldp q1, q0, [x0]
+; SVE-FIXED-NEXT: zip1 z2.d, z1.d, z0.d
+; SVE-FIXED-NEXT: trn2 z0.d, z1.d, z0.d
+; SVE-FIXED-NEXT: // fake_use: $q2 $z2
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
; SVE-FIXED-NEXT: ret
%load = load <4 x ptr>, ptr %ptr, align 8
%deinterleave = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> %load)
%extract1 = extractvalue { <2 x ptr>, <2 x ptr> } %deinterleave, 0
%extract2 = extractvalue { <2 x ptr>, <2 x ptr> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<2 x ptr> %extract1)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %extract2)
ret void
}
@@ -364,17 +543,105 @@ define void @interleave_ptr_factor2(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
}
define void @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
-; NEON-LABEL: deinterleave_wide_i16_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: deinterleave_wide_i16_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.8h, v1.8h }, [x0], #32
+; NEON-IAENABLED-NEXT: ld2 { v2.8h, v3.8h }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q2_q3
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: deinterleave_wide_i16_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: ldp q3, q2, [x0, #32]
+; NEON-IADISABLED-NEXT: uzp1 v4.8h, v1.8h, v0.8h
+; NEON-IADISABLED-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; NEON-IADISABLED-NEXT: uzp2 v5.8h, v3.8h, v2.8h
+; NEON-IADISABLED-NEXT: uzp1 v1.8h, v3.8h, v2.8h
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q5
+; NEON-IADISABLED-NEXT: ret
;
; SVE-FIXED-LABEL: deinterleave_wide_i16_factor2:
; SVE-FIXED: // %bb.0:
+; SVE-FIXED-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
+; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16
+; SVE-FIXED-NEXT: .cfi_offset b8, -16
+; SVE-FIXED-NEXT: ldp q0, q1, [x0, #32]
+; SVE-FIXED-NEXT: ldp q2, q3, [x0]
+; SVE-FIXED-NEXT: mov z4.h, z1.h[7]
+; SVE-FIXED-NEXT: mov z5.h, z1.h[5]
+; SVE-FIXED-NEXT: mov z6.h, z1.h[3]
+; SVE-FIXED-NEXT: mov z7.h, z1.h[1]
+; SVE-FIXED-NEXT: mov z16.h, z0.h[7]
+; SVE-FIXED-NEXT: mov z17.h, z0.h[5]
+; SVE-FIXED-NEXT: mov z18.h, z0.h[3]
+; SVE-FIXED-NEXT: mov z19.h, z0.h[1]
+; SVE-FIXED-NEXT: mov z20.h, z3.h[7]
+; SVE-FIXED-NEXT: mov z21.h, z3.h[5]
+; SVE-FIXED-NEXT: mov z22.h, z3.h[3]
+; SVE-FIXED-NEXT: mov z23.h, z3.h[1]
+; SVE-FIXED-NEXT: mov z24.h, z2.h[7]
+; SVE-FIXED-NEXT: mov z25.h, z2.h[5]
+; SVE-FIXED-NEXT: zip1 z4.h, z5.h, z4.h
+; SVE-FIXED-NEXT: zip1 z5.h, z7.h, z6.h
+; SVE-FIXED-NEXT: zip1 z6.h, z17.h, z16.h
+; SVE-FIXED-NEXT: zip1 z7.h, z19.h, z18.h
+; SVE-FIXED-NEXT: zip1 z16.h, z21.h, z20.h
+; SVE-FIXED-NEXT: zip1 z17.h, z23.h, z22.h
+; SVE-FIXED-NEXT: mov z19.h, z2.h[3]
+; SVE-FIXED-NEXT: zip1 z18.h, z25.h, z24.h
+; SVE-FIXED-NEXT: mov z20.h, z2.h[1]
+; SVE-FIXED-NEXT: mov z21.h, z1.h[6]
+; SVE-FIXED-NEXT: mov z22.h, z1.h[4]
+; SVE-FIXED-NEXT: mov z24.h, z3.h[6]
+; SVE-FIXED-NEXT: mov z25.h, z3.h[4]
+; SVE-FIXED-NEXT: mov z26.h, z3.h[2]
+; SVE-FIXED-NEXT: mov z27.h, z2.h[6]
+; SVE-FIXED-NEXT: mov z28.h, z2.h[4]
+; SVE-FIXED-NEXT: mov z29.h, z2.h[2]
+; SVE-FIXED-NEXT: mov z23.h, z1.h[2]
+; SVE-FIXED-NEXT: mov z30.h, z0.h[6]
+; SVE-FIXED-NEXT: mov z31.h, z0.h[4]
+; SVE-FIXED-NEXT: mov z8.h, z0.h[2]
+; SVE-FIXED-NEXT: zip1 z19.h, z20.h, z19.h
+; SVE-FIXED-NEXT: zip1 z20.h, z22.h, z21.h
+; SVE-FIXED-NEXT: zip1 z21.h, z25.h, z24.h
+; SVE-FIXED-NEXT: zip1 z3.h, z3.h, z26.h
+; SVE-FIXED-NEXT: zip1 z22.h, z28.h, z27.h
+; SVE-FIXED-NEXT: zip1 z2.h, z2.h, z29.h
+; SVE-FIXED-NEXT: zip1 z1.h, z1.h, z23.h
+; SVE-FIXED-NEXT: zip1 z23.h, z31.h, z30.h
+; SVE-FIXED-NEXT: zip1 z0.h, z0.h, z8.h
+; SVE-FIXED-NEXT: zip1 z4.s, z5.s, z4.s
+; SVE-FIXED-NEXT: zip1 z3.s, z3.s, z21.s
+; SVE-FIXED-NEXT: zip1 z5.s, z7.s, z6.s
+; SVE-FIXED-NEXT: zip1 z6.s, z17.s, z16.s
+; SVE-FIXED-NEXT: zip1 z2.s, z2.s, z22.s
+; SVE-FIXED-NEXT: zip1 z7.s, z19.s, z18.s
+; SVE-FIXED-NEXT: zip1 z1.s, z1.s, z20.s
+; SVE-FIXED-NEXT: zip1 z0.s, z0.s, z23.s
+; SVE-FIXED-NEXT: zip1 z2.d, z2.d, z3.d
+; SVE-FIXED-NEXT: zip1 z3.d, z5.d, z4.d
+; SVE-FIXED-NEXT: zip1 z4.d, z7.d, z6.d
+; SVE-FIXED-NEXT: zip1 z0.d, z0.d, z1.d
+; SVE-FIXED-NEXT: // fake_use: $q2 $z2
+; SVE-FIXED-NEXT: // fake_use: $q0 $z0
+; SVE-FIXED-NEXT: // fake_use: $q4 $z4
+; SVE-FIXED-NEXT: // fake_use: $q3 $z3
+; SVE-FIXED-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload
; SVE-FIXED-NEXT: ret
%load = load <32 x i16>, ptr %ptr, align 2
%deinterleave = tail call { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16> %load)
%extract1 = extractvalue { <16 x i16>, <16 x i16> } %deinterleave, 0
%extract2 = extractvalue { <16 x i16>, <16 x i16> } %deinterleave, 1
+ tail call void (...) @llvm.fake.use(<16 x i16> %extract1)
+ tail call void (...) @llvm.fake.use(<16 x i16> %extract2)
ret void
}
@@ -442,24 +709,6 @@ define void @interleave_wide_ptr_factor2(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
ret void
}
-declare { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8>)
-declare { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16>)
-declare { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32>)
-declare { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64>)
-declare { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float>)
-declare { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double>)
-declare { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr>)
-declare { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16>)
-
-declare <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8>, <16 x i8>)
-declare <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16>, <8 x i16>)
-declare <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32>, <4 x i32>)
-declare <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64>, <2 x i64>)
-declare <8 x float> @llvm.vector.interleave2.v8f32(<4 x float>, <4 x float>)
-declare <4 x double> @llvm.vector.interleave2.v4f64(<2 x double>, <2 x double>)
-declare <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr>, <2 x ptr>)
-declare <16 x ptr> @llvm.vector.interleave2.v16p0(<8 x ptr>, <8 x ptr>)
-
define void @interleave_double_factor2_intrinsic(ptr %ptr, <2 x double> %l, <2 x double> %r) {
; NEON-IAENABLED-LABEL: interleave_double_factor2_intrinsic:
; NEON-IAENABLED: // %bb.0:
@@ -777,5 +1026,6 @@ define void @interleave_wide_ptr_factor2_intrinsic(ptr %ptr, <8 x ptr> %l, <8 x
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NEON: {{.*}}
; SVE-FIXED-IADISABLED: {{.*}}
; SVE-FIXED-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
deleted file mode 100644
index bdb64c98d33e4..0000000000000
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user-inseltpoison.ll
+++ /dev/null
@@ -1,95 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
-
-define void @extract_user_basic(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_basic:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 2
- br label %if.merge
-
-if.merge:
- ret void
-}
-
-define void @extract_user_multi(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_multi:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
- br label %if.merge
-
-if.merge:
- %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
- ret void
-}
-
-define void @extract_user_multi_no_dom(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_multi_no_dom:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
- br label %if.merge
-
-if.merge:
- ret void
-}
-
-define void @extract_user_wrong_const_index(ptr %ptr) {
-; CHECK-LABEL: extract_user_wrong_const_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 1
- ret void
-}
-
-define void @extract_user_undef_index(ptr %ptr) {
-; CHECK-LABEL: extract_user_undef_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 undef
- ret void
-}
-
-define void @extract_user_var_index(ptr %ptr, i32 %i) {
-; CHECK-LABEL: extract_user_var_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
- ret void
-}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-IADISABLED: {{.*}}
-; CHECK-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
deleted file mode 100644
index 463ad52a3196d..0000000000000
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses-extract-user.ll
+++ /dev/null
@@ -1,95 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
-
-define void @extract_user_basic(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_basic:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 2
- br label %if.merge
-
-if.merge:
- ret void
-}
-
-define void @extract_user_multi(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_multi:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
- br label %if.merge
-
-if.merge:
- %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
- ret void
-}
-
-define void @extract_user_multi_no_dom(ptr %ptr, i1 %c) {
-; CHECK-LABEL: extract_user_multi_no_dom:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
- br i1 %c, label %if.then, label %if.merge
-
-if.then:
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
- br label %if.merge
-
-if.merge:
- ret void
-}
-
-define void @extract_user_wrong_const_index(ptr %ptr) {
-; CHECK-LABEL: extract_user_wrong_const_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 1
- ret void
-}
-
-define void @extract_user_undef_index(ptr %ptr) {
-; CHECK-LABEL: extract_user_undef_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 undef
- ret void
-}
-
-define void @extract_user_var_index(ptr %ptr, i32 %i) {
-; CHECK-LABEL: extract_user_var_index:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ret
-entry:
- %interleaved.vec = load <8 x i32>, ptr %ptr, align 8
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
- %e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
- ret void
-}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-IADISABLED: {{.*}}
-; CHECK-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index a706155c385a6..bedce9ec6a86e 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -1,54 +1,193 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
-; RUN: llc < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IADISABLED
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=NEON,NEON-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IADISABLED
define void @load_factor2(ptr %ptr) {
-; NEON-LABEL: load_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.8b, v1.8b }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $d0
+; NEON-IAENABLED-NEXT: // fake_use: $d1 $d0_d1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldr q0, [x0]
+; NEON-IADISABLED-NEXT: xtn v1.8b, v0.8h
+; NEON-IADISABLED-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; NEON-IADISABLED-NEXT: // fake_use: $d1
+; NEON-IADISABLED-NEXT: // fake_use: $d0 $q0
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor2:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldrb w2, [x0]
+; NO_NEON-NEXT: ldrb w8, [x0, #15]
+; NO_NEON-NEXT: ldrb w9, [x0, #13]
+; NO_NEON-NEXT: ldrb w10, [x0, #11]
+; NO_NEON-NEXT: ldrb w11, [x0, #9]
+; NO_NEON-NEXT: ldrb w12, [x0, #7]
+; NO_NEON-NEXT: ldrb w13, [x0, #5]
+; NO_NEON-NEXT: ldrb w14, [x0, #3]
+; NO_NEON-NEXT: ldrb w15, [x0, #1]
+; NO_NEON-NEXT: ldrb w16, [x0, #14]
+; NO_NEON-NEXT: ldrb w17, [x0, #12]
+; NO_NEON-NEXT: ldrb w18, [x0, #10]
+; NO_NEON-NEXT: ldrb w1, [x0, #8]
+; NO_NEON-NEXT: ldrb w3, [x0, #6]
+; NO_NEON-NEXT: ldrb w4, [x0, #4]
+; NO_NEON-NEXT: ldrb w0, [x0, #2]
+; NO_NEON-NEXT: // fake_use: $w2
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w4
+; NO_NEON-NEXT: // fake_use: $w3
+; NO_NEON-NEXT: // fake_use: $w1
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i8>, ptr %ptr, align 4
%v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ tail call void (...) @llvm.fake.use(<8 x i8> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i8> %v1)
ret void
}
define void @load_factor3(ptr %ptr) {
-; NEON-LABEL: load_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2 $q0_q1_q2
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q0, q2, [x0]
+; NEON-IADISABLED-NEXT: ldr q4, [x0, #32]
+; NEON-IADISABLED-NEXT: mov v1.16b, v0.16b
+; NEON-IADISABLED-NEXT: rev64 v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: mov v1.s[1], v0.s[3]
+; NEON-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; NEON-IADISABLED-NEXT: mov v1.s[2], v2.s[2]
+; NEON-IADISABLED-NEXT: mov v2.s[0], v0.s[2]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v4.s[2]
+; NEON-IADISABLED-NEXT: mov v2.s[2], v4.s[0]
+; NEON-IADISABLED-NEXT: mov v1.s[3], v4.s[1]
+; NEON-IADISABLED-NEXT: mov v2.s[3], v4.s[3]
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: // fake_use: $q3
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor3:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w15, w14, [x0]
+; NO_NEON-NEXT: ldp w9, w8, [x0, #40]
+; NO_NEON-NEXT: ldp w11, w10, [x0, #16]
+; NO_NEON-NEXT: ldp w12, w13, [x0, #32]
+; NO_NEON-NEXT: ldp w17, w16, [x0, #24]
+; NO_NEON-NEXT: ldp w18, w0, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
%v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ tail call void (...) @llvm.fake.use(<4 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v2)
ret void
}
define void @load_factor4(ptr %ptr) {
-; NEON-LABEL: load_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q2 $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0, #32]
+; NEON-IADISABLED-NEXT: ldp q4, q3, [x0]
+; NEON-IADISABLED-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: trn1 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: zip1 v5.4s, v4.4s, v3.4s
+; NEON-IADISABLED-NEXT: trn2 v7.4s, v4.4s, v3.4s
+; NEON-IADISABLED-NEXT: ext v6.16b, v1.16b, v2.16b, #8
+; NEON-IADISABLED-NEXT: zip2 v1.4s, v4.4s, v3.4s
+; NEON-IADISABLED-NEXT: mov v7.d[1], v2.d[1]
+; NEON-IADISABLED-NEXT: mov v5.d[1], v6.d[1]
+; NEON-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; NEON-IADISABLED-NEXT: // fake_use: $q5
+; NEON-IADISABLED-NEXT: // fake_use: $q7
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor4:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w12, w8, [x0, #52]
+; NO_NEON-NEXT: ldp w16, w9, [x0, #36]
+; NO_NEON-NEXT: ldp w18, w10, [x0, #20]
+; NO_NEON-NEXT: ldp w14, w11, [x0, #4]
+; NO_NEON-NEXT: ldr w15, [x0]
+; NO_NEON-NEXT: ldr w13, [x0, #48]
+; NO_NEON-NEXT: ldr w17, [x0, #32]
+; NO_NEON-NEXT: ldr w0, [x0, #16]
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ tail call void (...) @llvm.fake.use(<4 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v2)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v2)
ret void
}
@@ -151,7 +290,7 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
; NO_NEON-NEXT: stp x10, x9, [x0]
; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -217,47 +356,130 @@ define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
}
define void @load_ptrvec_factor2(ptr %ptr) {
-; NEON-LABEL: load_ptrvec_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_ptrvec_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_ptrvec_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: zip1 v2.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_ptrvec_factor2:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x8, x9, [x0]
+; NO_NEON-NEXT: ldp x11, x10, [x0, #16]
+; NO_NEON-NEXT: // fake_use: $x8
+; NO_NEON-NEXT: // fake_use: $x11
+; NO_NEON-NEXT: // fake_use: $x9
+; NO_NEON-NEXT: // fake_use: $x10
; NO_NEON-NEXT: ret
%interleaved.vec = load <4 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 1, i32 3>
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v1)
ret void
}
define void @load_ptrvec_factor3(ptr %ptr) {
-; NEON-LABEL: load_ptrvec_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_ptrvec_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2 $q0_q1_q2
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_ptrvec_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0, #16]
+; NEON-IADISABLED-NEXT: ldr q2, [x0]
+; NEON-IADISABLED-NEXT: ext v3.16b, v2.16b, v0.16b, #8
+; NEON-IADISABLED-NEXT: mov v2.d[1], v1.d[1]
+; NEON-IADISABLED-NEXT: mov v1.d[1], v0.d[1]
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q3
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_ptrvec_factor3:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x9, x8, [x0]
+; NO_NEON-NEXT: ldp x11, x10, [x0, #32]
+; NO_NEON-NEXT: ldp x12, x13, [x0, #16]
+; NO_NEON-NEXT: // fake_use: $x9
+; NO_NEON-NEXT: // fake_use: $x13
+; NO_NEON-NEXT: // fake_use: $x8
+; NO_NEON-NEXT: // fake_use: $x11
+; NO_NEON-NEXT: // fake_use: $x12
+; NO_NEON-NEXT: // fake_use: $x10
; NO_NEON-NEXT: ret
%interleaved.vec = load <6 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 0, i32 3>
%v1 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 1, i32 4>
%v2 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 2, i32 5>
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v1)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v2)
ret void
}
define void @load_ptrvec_factor4(ptr %ptr) {
-; NEON-LABEL: load_ptrvec_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_ptrvec_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_ptrvec_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q0, q3, [x0]
+; NEON-IADISABLED-NEXT: ldp q1, q2, [x0, #32]
+; NEON-IADISABLED-NEXT: zip1 v4.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; NEON-IADISABLED-NEXT: zip1 v1.2d, v3.2d, v2.2d
+; NEON-IADISABLED-NEXT: zip2 v2.2d, v3.2d, v2.2d
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_ptrvec_factor4:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x11, x10, [x0]
+; NO_NEON-NEXT: ldp x9, x8, [x0, #48]
+; NO_NEON-NEXT: ldp x13, x12, [x0, #16]
+; NO_NEON-NEXT: ldp x15, x14, [x0, #32]
+; NO_NEON-NEXT: // fake_use: $x11
+; NO_NEON-NEXT: // fake_use: $x15
+; NO_NEON-NEXT: // fake_use: $x10
+; NO_NEON-NEXT: // fake_use: $x14
+; NO_NEON-NEXT: // fake_use: $x13
+; NO_NEON-NEXT: // fake_use: $x9
+; NO_NEON-NEXT: // fake_use: $x12
+; NO_NEON-NEXT: // fake_use: $x8
; NO_NEON-NEXT: ret
%interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 0, i32 4>
%v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 1, i32 5>
%v2 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 2, i32 6>
%v3 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 3, i32 7>
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v1)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v2)
+ tail call void (...) @llvm.fake.use(<2 x ptr> %v3)
ret void
}
@@ -311,7 +533,7 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
; NO_NEON-NEXT: stp x2, x4, [x0]
; NO_NEON-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %s1 = shufflevector <2 x ptr> %v2, <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %s1 = shufflevector <2 x ptr> %v2, <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
%interleaved.vec = shufflevector <4 x ptr> %s0, <4 x ptr> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
store <6 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -354,47 +576,155 @@ define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
}
define void @load_undef_mask_factor2(ptr %ptr) {
-; NEON-LABEL: load_undef_mask_factor2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_undef_mask_factor2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_undef_mask_factor2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_undef_mask_factor2:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldp w11, w10, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
%interleaved.vec = load <8 x i32>, ptr %ptr, align 4
- %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
- %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+ %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 poison, i32 2, i32 poison, i32 6>
+ %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 7>
+ tail call void (...) @llvm.fake.use(<4 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v1)
ret void
}
define void @load_undef_mask_factor3(ptr %ptr) {
-; NEON-LABEL: load_undef_mask_factor3:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_undef_mask_factor3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2 $q0_q1_q2
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_undef_mask_factor3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q0, q1, [x0]
+; NEON-IADISABLED-NEXT: mov v2.16b, v0.16b
+; NEON-IADISABLED-NEXT: rev64 v3.4s, v1.4s
+; NEON-IADISABLED-NEXT: mov v2.s[1], v0.s[3]
+; NEON-IADISABLED-NEXT: mov v3.s[0], v0.s[1]
+; NEON-IADISABLED-NEXT: dup v0.4s, v0.s[2]
+; NEON-IADISABLED-NEXT: mov v2.s[2], v1.s[2]
+; NEON-IADISABLED-NEXT: ldr q1, [x0, #32]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v1.s[2]
+; NEON-IADISABLED-NEXT: mov v2.s[3], v1.s[1]
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q3
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_undef_mask_factor3:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w12, w11, [x0]
+; NO_NEON-NEXT: ldp w16, w8, [x0, #12]
+; NO_NEON-NEXT: ldp w10, w9, [x0, #36]
+; NO_NEON-NEXT: ldp w14, w13, [x0, #24]
+; NO_NEON-NEXT: ldr w15, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
- %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+ %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
+ tail call void (...) @llvm.fake.use(<4 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v2)
ret void
}
define void @load_undef_mask_factor4(ptr %ptr) {
-; NEON-LABEL: load_undef_mask_factor4:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_undef_mask_factor4:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_undef_mask_factor4:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: uzp2 v2.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: zip1 v3.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: trn2 v4.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v1.4s, v2.4s, v1.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q3
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_undef_mask_factor4:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w11, w10, [x0]
+; NO_NEON-NEXT: ldp w9, w8, [x0, #24]
+; NO_NEON-NEXT: ldp w13, w12, [x0, #8]
+; NO_NEON-NEXT: ldp w15, w14, [x0, #16]
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
- %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
- %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
- %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
- %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+ %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
+ %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+ %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 poison, i32 poison>
+ %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 poison, i32 poison>
+ tail call void (...) @llvm.fake.use(<4 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v2)
+ tail call void (...) @llvm.fake.use(<4 x i32> %v3)
ret void
}
@@ -418,7 +748,7 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
; NO_NEON-NEXT: stp w3, w7, [x0, #16]
; NO_NEON-NEXT: stp w4, w8, [x0, #24]
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+ %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -469,8 +799,8 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: stp x9, x11, [x0]
; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
- %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+ %s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+ %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -524,7 +854,7 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+ %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
store <16 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -532,13 +862,23 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
define void @load_illegal_factor2(ptr %ptr) nounwind {
; NEON-LABEL: load_illegal_factor2:
; NEON: // %bb.0:
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; NEON-NEXT: // fake_use: $q0
; NEON-NEXT: ret
;
; NO_NEON-LABEL: load_illegal_factor2:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldr s0, [x0]
+; NO_NEON-NEXT: ldr s1, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $s0
+; NO_NEON-NEXT: // fake_use: $s1
+; NO_NEON-NEXT: // fake_use: $s0
+; NO_NEON-NEXT: // fake_use: $s0
; NO_NEON-NEXT: ret
%interleaved.vec = load <3 x float>, ptr %ptr, align 16
- %v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+ %v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
+ tail call void (...) @llvm.fake.use(<3 x float> %v0)
ret void
}
@@ -556,7 +896,7 @@ define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <3 x float> %v0, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+ %interleaved.vec = shufflevector <3 x float> %v0, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
store <3 x float> %interleaved.vec, ptr %ptr, align 16
ret void
}
@@ -649,7 +989,7 @@ define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: str w10, [x0, #4]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -693,7 +1033,7 @@ define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: stp w5, w9, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -735,7 +1075,7 @@ define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: str w5, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -769,7 +1109,7 @@ define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: str w9, [x0, #28]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -884,7 +1224,7 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -930,7 +1270,7 @@ define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -980,7 +1320,7 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: stp x9, x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -1025,7 +1365,7 @@ define void @store_general_mask_factor3_negativestart(ptr %ptr, <32 x i32> %v0,
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
- %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+ %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
@@ -1050,67 +1390,430 @@ define void @no_interleave(<4 x float> %a0) {
; NO_NEON-NEXT: stp s0, s3, [x8]
; NO_NEON-NEXT: str s3, [x8, #8]
; NO_NEON-NEXT: ret
- %v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 3, i32 7, i32 undef>
+ %v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 3, i32 7, i32 poison>
store <4 x float> %v0, ptr @g, align 16
ret void
}
define void @load_factor2_wide2(ptr %ptr) {
-; NEON-LABEL: load_factor2_wide2:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor2_wide2:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; NEON-IAENABLED-NEXT: ld2 { v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q2_q3
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor2_wide2:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: ldp q3, q2, [x0, #32]
+; NEON-IADISABLED-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q5
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor2_wide2:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp w1, w18, [x0]
+; NO_NEON-NEXT: ldp w9, w8, [x0, #56]
+; NO_NEON-NEXT: ldp w11, w10, [x0, #48]
+; NO_NEON-NEXT: ldp w13, w12, [x0, #40]
+; NO_NEON-NEXT: ldp w15, w14, [x0, #32]
+; NO_NEON-NEXT: ldp w17, w16, [x0, #24]
+; NO_NEON-NEXT: ldp w3, w2, [x0, #16]
+; NO_NEON-NEXT: ldp w0, w4, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w1
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w3
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w4
+; NO_NEON-NEXT: // fake_use: $w2
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ tail call void (...) @llvm.fake.use(<8 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v1)
ret void
}
define void @load_factor2_wide3(ptr %ptr) {
-; NEON-LABEL: load_factor2_wide3:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor2_wide3:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: add x8, x0, #64
+; NEON-IAENABLED-NEXT: ld2 { v0.4s, v1.4s }, [x0], #32
+; NEON-IAENABLED-NEXT: ld2 { v2.4s, v3.4s }, [x0]
+; NEON-IAENABLED-NEXT: ld2 { v4.4s, v5.4s }, [x8]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q4
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q2_q3
+; NEON-IAENABLED-NEXT: // fake_use: $q5 $q4_q5
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor2_wide3:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0]
+; NEON-IADISABLED-NEXT: ldp q3, q2, [x0, #32]
+; NEON-IADISABLED-NEXT: ldp q5, q4, [x0, #64]
+; NEON-IADISABLED-NEXT: uzp1 v16.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp1 v7.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: uzp1 v6.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: // fake_use: $q16
+; NEON-IADISABLED-NEXT: // fake_use: $q7
+; NEON-IADISABLED-NEXT: // fake_use: $q6
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q0
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor2_wide3:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x23, [sp, #-48]! // 8-byte Folded Spill
+; NO_NEON-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 48
+; NO_NEON-NEXT: .cfi_offset w19, -8
+; NO_NEON-NEXT: .cfi_offset w20, -16
+; NO_NEON-NEXT: .cfi_offset w21, -24
+; NO_NEON-NEXT: .cfi_offset w22, -32
+; NO_NEON-NEXT: .cfi_offset w23, -48
+; NO_NEON-NEXT: ldp w20, w19, [x0]
+; NO_NEON-NEXT: ldp w9, w8, [x0, #88]
+; NO_NEON-NEXT: ldp w11, w10, [x0, #80]
+; NO_NEON-NEXT: ldp w13, w12, [x0, #72]
+; NO_NEON-NEXT: ldp w15, w14, [x0, #64]
+; NO_NEON-NEXT: ldp w17, w16, [x0, #56]
+; NO_NEON-NEXT: ldp w1, w18, [x0, #48]
+; NO_NEON-NEXT: ldp w3, w2, [x0, #40]
+; NO_NEON-NEXT: ldp w5, w4, [x0, #32]
+; NO_NEON-NEXT: ldp w7, w6, [x0, #24]
+; NO_NEON-NEXT: ldp w22, w21, [x0, #16]
+; NO_NEON-NEXT: ldp w0, w23, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w20
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w22
+; NO_NEON-NEXT: // fake_use: $w7
+; NO_NEON-NEXT: // fake_use: $w5
+; NO_NEON-NEXT: // fake_use: $w3
+; NO_NEON-NEXT: // fake_use: $w1
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w19
+; NO_NEON-NEXT: // fake_use: $w23
+; NO_NEON-NEXT: // fake_use: $w21
+; NO_NEON-NEXT: // fake_use: $w6
+; NO_NEON-NEXT: // fake_use: $w4
+; NO_NEON-NEXT: // fake_use: $w2
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
+ tail call void (...) @llvm.fake.use(<12 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<12 x i32> %v1)
ret void
}
define void @load_factor3_wide(ptr %ptr) {
-; NEON-LABEL: load_factor3_wide:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor3_wide:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; NEON-IAENABLED-NEXT: ld3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q3
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q4
+; NEON-IAENABLED-NEXT: // fake_use: $q2 $q0_q1_q2
+; NEON-IAENABLED-NEXT: // fake_use: $q5 $q3_q4_q5
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor3_wide:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q0, q4, [x0]
+; NEON-IADISABLED-NEXT: ldr q16, [x0, #32]
+; NEON-IADISABLED-NEXT: ldp q1, q5, [x0, #48]
+; NEON-IADISABLED-NEXT: ldr q17, [x0, #80]
+; NEON-IADISABLED-NEXT: mov v3.16b, v0.16b
+; NEON-IADISABLED-NEXT: rev64 v7.4s, v4.4s
+; NEON-IADISABLED-NEXT: mov v2.16b, v1.16b
+; NEON-IADISABLED-NEXT: rev64 v6.4s, v5.4s
+; NEON-IADISABLED-NEXT: mov v3.s[1], v0.s[3]
+; NEON-IADISABLED-NEXT: mov v2.s[1], v1.s[3]
+; NEON-IADISABLED-NEXT: mov v7.s[0], v0.s[1]
+; NEON-IADISABLED-NEXT: mov v6.s[0], v1.s[1]
+; NEON-IADISABLED-NEXT: mov v3.s[2], v4.s[2]
+; NEON-IADISABLED-NEXT: mov v4.s[0], v0.s[2]
+; NEON-IADISABLED-NEXT: mov v2.s[2], v5.s[2]
+; NEON-IADISABLED-NEXT: mov v5.s[0], v1.s[2]
+; NEON-IADISABLED-NEXT: mov v7.s[3], v16.s[2]
+; NEON-IADISABLED-NEXT: mov v6.s[3], v17.s[2]
+; NEON-IADISABLED-NEXT: mov v4.s[2], v16.s[0]
+; NEON-IADISABLED-NEXT: mov v3.s[3], v16.s[1]
+; NEON-IADISABLED-NEXT: mov v5.s[2], v17.s[0]
+; NEON-IADISABLED-NEXT: mov v2.s[3], v17.s[1]
+; NEON-IADISABLED-NEXT: mov v4.s[3], v16.s[3]
+; NEON-IADISABLED-NEXT: mov v5.s[3], v17.s[3]
+; NEON-IADISABLED-NEXT: // fake_use: $q3
+; NEON-IADISABLED-NEXT: // fake_use: $q2
+; NEON-IADISABLED-NEXT: // fake_use: $q7
+; NEON-IADISABLED-NEXT: // fake_use: $q6
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q5
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor3_wide:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: str x23, [sp, #-48]! // 8-byte Folded Spill
+; NO_NEON-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 48
+; NO_NEON-NEXT: .cfi_offset w19, -8
+; NO_NEON-NEXT: .cfi_offset w20, -16
+; NO_NEON-NEXT: .cfi_offset w21, -24
+; NO_NEON-NEXT: .cfi_offset w22, -32
+; NO_NEON-NEXT: .cfi_offset w23, -48
+; NO_NEON-NEXT: ldp w20, w19, [x0]
+; NO_NEON-NEXT: ldp w9, w8, [x0, #88]
+; NO_NEON-NEXT: ldp w11, w10, [x0, #64]
+; NO_NEON-NEXT: ldp w13, w12, [x0, #40]
+; NO_NEON-NEXT: ldp w15, w14, [x0, #16]
+; NO_NEON-NEXT: ldp w16, w17, [x0, #80]
+; NO_NEON-NEXT: ldp w1, w18, [x0, #72]
+; NO_NEON-NEXT: ldp w2, w3, [x0, #56]
+; NO_NEON-NEXT: ldp w5, w4, [x0, #48]
+; NO_NEON-NEXT: ldp w6, w7, [x0, #32]
+; NO_NEON-NEXT: ldp w22, w21, [x0, #24]
+; NO_NEON-NEXT: ldp w23, w0, [x0, #8]
+; NO_NEON-NEXT: // fake_use: $w20
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w22
+; NO_NEON-NEXT: // fake_use: $w7
+; NO_NEON-NEXT: // fake_use: $w5
+; NO_NEON-NEXT: // fake_use: $w3
+; NO_NEON-NEXT: // fake_use: $w1
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w19
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w21
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w4
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: // fake_use: $w23
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w6
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w2
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
%v2 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ tail call void (...) @llvm.fake.use(<8 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v2)
ret void
}
define void @load_factor4_wide(ptr %ptr) {
-; NEON-LABEL: load_factor4_wide:
-; NEON: // %bb.0:
-; NEON-NEXT: ret
+; NEON-IAENABLED-LABEL: load_factor4_wide:
+; NEON-IAENABLED: // %bb.0:
+; NEON-IAENABLED-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; NEON-IAENABLED-NEXT: ld4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-IAENABLED-NEXT: // fake_use: $q0
+; NEON-IAENABLED-NEXT: // fake_use: $q4
+; NEON-IAENABLED-NEXT: // fake_use: $q1
+; NEON-IAENABLED-NEXT: // fake_use: $q5
+; NEON-IAENABLED-NEXT: // fake_use: $q2
+; NEON-IAENABLED-NEXT: // fake_use: $q6
+; NEON-IAENABLED-NEXT: // fake_use: $q3 $q0_q1_q2_q3
+; NEON-IAENABLED-NEXT: // fake_use: $q7 $q4_q5_q6_q7
+; NEON-IAENABLED-NEXT: ret
+;
+; NEON-IADISABLED-LABEL: load_factor4_wide:
+; NEON-IADISABLED: // %bb.0:
+; NEON-IADISABLED-NEXT: ldp q1, q0, [x0, #32]
+; NEON-IADISABLED-NEXT: ldp q3, q2, [x0, #96]
+; NEON-IADISABLED-NEXT: ldp q5, q4, [x0, #64]
+; NEON-IADISABLED-NEXT: zip1 v17.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: ldp q7, q6, [x0]
+; NEON-IADISABLED-NEXT: zip1 v16.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: trn1 v23.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: trn1 v24.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: uzp2 v20.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v18.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: trn2 v22.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip1 v19.4s, v7.4s, v6.4s
+; NEON-IADISABLED-NEXT: uzp2 v21.4s, v7.4s, v6.4s
+; NEON-IADISABLED-NEXT: zip2 v2.4s, v3.4s, v2.4s
+; NEON-IADISABLED-NEXT: ext v25.16b, v1.16b, v17.16b, #8
+; NEON-IADISABLED-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; NEON-IADISABLED-NEXT: trn2 v1.4s, v7.4s, v6.4s
+; NEON-IADISABLED-NEXT: ext v3.16b, v3.16b, v16.16b, #8
+; NEON-IADISABLED-NEXT: zip2 v4.4s, v5.4s, v4.4s
+; NEON-IADISABLED-NEXT: zip2 v6.4s, v7.4s, v6.4s
+; NEON-IADISABLED-NEXT: uzp2 v5.4s, v20.4s, v5.4s
+; NEON-IADISABLED-NEXT: mov v22.d[1], v16.d[1]
+; NEON-IADISABLED-NEXT: uzp2 v7.4s, v21.4s, v7.4s
+; NEON-IADISABLED-NEXT: mov v19.d[1], v25.d[1]
+; NEON-IADISABLED-NEXT: mov v1.d[1], v17.d[1]
+; NEON-IADISABLED-NEXT: mov v18.d[1], v3.d[1]
+; NEON-IADISABLED-NEXT: mov v4.d[1], v23.d[1]
+; NEON-IADISABLED-NEXT: mov v6.d[1], v24.d[1]
+; NEON-IADISABLED-NEXT: mov v5.d[1], v2.d[1]
+; NEON-IADISABLED-NEXT: mov v7.d[1], v0.d[1]
+; NEON-IADISABLED-NEXT: // fake_use: $q19
+; NEON-IADISABLED-NEXT: // fake_use: $q18
+; NEON-IADISABLED-NEXT: // fake_use: $q1
+; NEON-IADISABLED-NEXT: // fake_use: $q22
+; NEON-IADISABLED-NEXT: // fake_use: $q6
+; NEON-IADISABLED-NEXT: // fake_use: $q4
+; NEON-IADISABLED-NEXT: // fake_use: $q7
+; NEON-IADISABLED-NEXT: // fake_use: $q5
+; NEON-IADISABLED-NEXT: ret
;
; NO_NEON-LABEL: load_factor4_wide:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: sub sp, sp, #112
+; NO_NEON-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill
+; NO_NEON-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; NO_NEON-NEXT: .cfi_def_cfa_offset 112
+; NO_NEON-NEXT: .cfi_offset w19, -8
+; NO_NEON-NEXT: .cfi_offset w20, -16
+; NO_NEON-NEXT: .cfi_offset w21, -24
+; NO_NEON-NEXT: .cfi_offset w22, -32
+; NO_NEON-NEXT: .cfi_offset w23, -40
+; NO_NEON-NEXT: .cfi_offset w24, -48
+; NO_NEON-NEXT: .cfi_offset w25, -56
+; NO_NEON-NEXT: .cfi_offset w26, -64
+; NO_NEON-NEXT: .cfi_offset w27, -72
+; NO_NEON-NEXT: .cfi_offset w28, -80
+; NO_NEON-NEXT: .cfi_offset w30, -88
+; NO_NEON-NEXT: .cfi_offset w29, -96
+; NO_NEON-NEXT: ldp w12, w8, [x0, #120]
+; NO_NEON-NEXT: ldp w28, w27, [x0]
+; NO_NEON-NEXT: ldp w13, w9, [x0, #104]
+; NO_NEON-NEXT: str w8, [sp, #12] // 4-byte Spill
+; NO_NEON-NEXT: ldp w14, w10, [x0, #88]
+; NO_NEON-NEXT: ldp w15, w11, [x0, #72]
+; NO_NEON-NEXT: ldp w17, w16, [x0, #56]
+; NO_NEON-NEXT: ldp w1, w18, [x0, #40]
+; NO_NEON-NEXT: ldp w3, w2, [x0, #24]
+; NO_NEON-NEXT: ldp w5, w4, [x0, #8]
+; NO_NEON-NEXT: ldp w7, w6, [x0, #112]
+; NO_NEON-NEXT: ldp w20, w19, [x0, #96]
+; NO_NEON-NEXT: ldp w22, w21, [x0, #80]
+; NO_NEON-NEXT: ldp w24, w23, [x0, #64]
+; NO_NEON-NEXT: ldp w26, w25, [x0, #48]
+; NO_NEON-NEXT: ldp w30, w29, [x0, #32]
+; NO_NEON-NEXT: ldp w0, w8, [x0, #16]
+; NO_NEON-NEXT: // fake_use: $w28
+; NO_NEON-NEXT: // fake_use: $w0
+; NO_NEON-NEXT: // fake_use: $w30
+; NO_NEON-NEXT: // fake_use: $w26
+; NO_NEON-NEXT: // fake_use: $w24
+; NO_NEON-NEXT: // fake_use: $w22
+; NO_NEON-NEXT: // fake_use: $w20
+; NO_NEON-NEXT: // fake_use: $w7
+; NO_NEON-NEXT: // fake_use: $w27
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: // fake_use: $w29
+; NO_NEON-NEXT: // fake_use: $w25
+; NO_NEON-NEXT: // fake_use: $w23
+; NO_NEON-NEXT: // fake_use: $w21
+; NO_NEON-NEXT: // fake_use: $w19
+; NO_NEON-NEXT: // fake_use: $w6
+; NO_NEON-NEXT: // fake_use: $w5
+; NO_NEON-NEXT: // fake_use: $w3
+; NO_NEON-NEXT: // fake_use: $w1
+; NO_NEON-NEXT: // fake_use: $w17
+; NO_NEON-NEXT: // fake_use: $w15
+; NO_NEON-NEXT: // fake_use: $w14
+; NO_NEON-NEXT: // fake_use: $w13
+; NO_NEON-NEXT: // fake_use: $w12
+; NO_NEON-NEXT: // fake_use: $w4
+; NO_NEON-NEXT: // fake_use: $w2
+; NO_NEON-NEXT: // fake_use: $w18
+; NO_NEON-NEXT: // fake_use: $w16
+; NO_NEON-NEXT: // fake_use: $w11
+; NO_NEON-NEXT: // fake_use: $w10
+; NO_NEON-NEXT: // fake_use: $w9
+; NO_NEON-NEXT: ldr w8, [sp, #12] // 4-byte Reload
+; NO_NEON-NEXT: // fake_use: $w8
+; NO_NEON-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; NO_NEON-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; NO_NEON-NEXT: add sp, sp, #112
; NO_NEON-NEXT: ret
%interleaved.vec = load <32 x i32>, ptr %ptr, align 4
%v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%v1 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
%v2 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
%v3 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+ tail call void (...) @llvm.fake.use(<8 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v2)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v3)
ret void
}
@@ -1249,7 +1952,7 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
; NO_NEON-NEXT: stp x8, x12, [x0]
; NO_NEON-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
store <24 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1309,53 +2012,54 @@ define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
; NO_NEON: // %bb.0:
; NO_NEON-NEXT: ldr w8, [sp, #192]
; NO_NEON-NEXT: ldr w9, [sp, #128]
-; NO_NEON-NEXT: ldr w10, [sp, #64]
; NO_NEON-NEXT: str w7, [x0, #96]
+; NO_NEON-NEXT: str w6, [x0, #80]
; NO_NEON-NEXT: str w8, [x0, #124]
-; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w8, [sp, #64]
; NO_NEON-NEXT: str w9, [x0, #120]
-; NO_NEON-NEXT: ldr w9, [sp, #184]
-; NO_NEON-NEXT: str w10, [x0, #116]
-; NO_NEON-NEXT: ldr w10, [sp, #120]
-; NO_NEON-NEXT: str w8, [x0, #112]
+; NO_NEON-NEXT: ldr w9, [sp]
+; NO_NEON-NEXT: str w8, [x0, #116]
+; NO_NEON-NEXT: ldr w8, [sp, #184]
+; NO_NEON-NEXT: str w9, [x0, #112]
+; NO_NEON-NEXT: ldr w9, [sp, #120]
+; NO_NEON-NEXT: str w8, [x0, #108]
; NO_NEON-NEXT: ldr w8, [sp, #56]
-; NO_NEON-NEXT: stp w10, w9, [x0, #104]
-; NO_NEON-NEXT: ldr w9, [sp, #176]
-; NO_NEON-NEXT: ldr w10, [sp, #48]
-; NO_NEON-NEXT: str w8, [x0, #100]
-; NO_NEON-NEXT: ldr w8, [sp, #112]
-; NO_NEON-NEXT: str w10, [x0, #84]
-; NO_NEON-NEXT: ldr w10, [sp, #40]
-; NO_NEON-NEXT: stp w8, w9, [x0, #88]
+; NO_NEON-NEXT: str w5, [x0, #64]
+; NO_NEON-NEXT: stp w8, w9, [x0, #100]
+; NO_NEON-NEXT: ldr w8, [sp, #176]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: str w4, [x0, #48]
+; NO_NEON-NEXT: str w8, [x0, #92]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w3, [x0, #32]
+; NO_NEON-NEXT: stp w8, w9, [x0, #84]
; NO_NEON-NEXT: ldr w8, [sp, #168]
; NO_NEON-NEXT: ldr w9, [sp, #104]
-; NO_NEON-NEXT: str w10, [x0, #68]
-; NO_NEON-NEXT: ldr w10, [sp, #32]
-; NO_NEON-NEXT: stp w9, w8, [x0, #72]
+; NO_NEON-NEXT: str w2, [x0, #16]
+; NO_NEON-NEXT: str w8, [x0, #76]
+; NO_NEON-NEXT: ldr w8, [sp, #40]
+; NO_NEON-NEXT: str w1, [x0]
+; NO_NEON-NEXT: stp w8, w9, [x0, #68]
; NO_NEON-NEXT: ldr w8, [sp, #160]
; NO_NEON-NEXT: ldr w9, [sp, #96]
-; NO_NEON-NEXT: str w10, [x0, #52]
-; NO_NEON-NEXT: ldr w10, [sp, #24]
-; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: str w8, [x0, #60]
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: stp w8, w9, [x0, #52]
; NO_NEON-NEXT: ldr w8, [sp, #152]
; NO_NEON-NEXT: ldr w9, [sp, #88]
-; NO_NEON-NEXT: str w10, [x0, #36]
-; NO_NEON-NEXT: ldr w10, [sp, #16]
-; NO_NEON-NEXT: stp w9, w8, [x0, #40]
+; NO_NEON-NEXT: str w8, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: stp w8, w9, [x0, #36]
; NO_NEON-NEXT: ldr w8, [sp, #144]
; NO_NEON-NEXT: ldr w9, [sp, #80]
-; NO_NEON-NEXT: str w10, [x0, #20]
-; NO_NEON-NEXT: ldr w10, [sp, #8]
-; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: str w8, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #16]
+; NO_NEON-NEXT: stp w8, w9, [x0, #20]
; NO_NEON-NEXT: ldr w8, [sp, #136]
; NO_NEON-NEXT: ldr w9, [sp, #72]
-; NO_NEON-NEXT: str w6, [x0, #80]
-; NO_NEON-NEXT: str w5, [x0, #64]
-; NO_NEON-NEXT: str w4, [x0, #48]
-; NO_NEON-NEXT: str w3, [x0, #32]
-; NO_NEON-NEXT: str w2, [x0, #16]
-; NO_NEON-NEXT: stp w9, w8, [x0, #8]
-; NO_NEON-NEXT: stp w1, w10, [x0]
+; NO_NEON-NEXT: str w8, [x0, #12]
+; NO_NEON-NEXT: ldr w8, [sp, #8]
+; NO_NEON-NEXT: stp w8, w9, [x0, #4]
; NO_NEON-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -1367,14 +2071,28 @@ define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
define void @load_factor2_fp128(ptr %ptr) {
; NEON-LABEL: load_factor2_fp128:
; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: ldp q3, q2, [x0, #32]
+; NEON-NEXT: // fake_use: $q0
+; NEON-NEXT: // fake_use: $q3
+; NEON-NEXT: // fake_use: $q1
+; NEON-NEXT: // fake_use: $q2
; NEON-NEXT: ret
;
; NO_NEON-LABEL: load_factor2_fp128:
; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp q0, q1, [x0]
+; NO_NEON-NEXT: ldp q3, q2, [x0, #32]
+; NO_NEON-NEXT: // fake_use: $q0
+; NO_NEON-NEXT: // fake_use: $q3
+; NO_NEON-NEXT: // fake_use: $q1
+; NO_NEON-NEXT: // fake_use: $q2
; NO_NEON-NEXT: ret
%interleaved.vec = load <4 x fp128>, ptr %ptr, align 16
%v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 1, i32 3>
+ tail call void (...) @llvm.fake.use(<2 x fp128> %v0)
+ tail call void (...) @llvm.fake.use(<2 x fp128> %v1)
ret void
}
@@ -1782,53 +2500,54 @@ define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1
; NO_NEON: // %bb.0:
; NO_NEON-NEXT: ldr w8, [sp, #192]
; NO_NEON-NEXT: ldr w9, [sp, #128]
-; NO_NEON-NEXT: ldr w10, [sp, #64]
; NO_NEON-NEXT: str w7, [x0, #96]
+; NO_NEON-NEXT: str w6, [x0, #80]
; NO_NEON-NEXT: str w8, [x0, #124]
-; NO_NEON-NEXT: ldr w8, [sp]
+; NO_NEON-NEXT: ldr w8, [sp, #64]
; NO_NEON-NEXT: str w9, [x0, #120]
-; NO_NEON-NEXT: ldr w9, [sp, #184]
-; NO_NEON-NEXT: str w10, [x0, #116]
-; NO_NEON-NEXT: ldr w10, [sp, #120]
-; NO_NEON-NEXT: str w8, [x0, #112]
+; NO_NEON-NEXT: ldr w9, [sp]
+; NO_NEON-NEXT: str w8, [x0, #116]
+; NO_NEON-NEXT: ldr w8, [sp, #184]
+; NO_NEON-NEXT: str w9, [x0, #112]
+; NO_NEON-NEXT: ldr w9, [sp, #120]
+; NO_NEON-NEXT: str w8, [x0, #108]
; NO_NEON-NEXT: ldr w8, [sp, #56]
-; NO_NEON-NEXT: stp w10, w9, [x0, #104]
-; NO_NEON-NEXT: ldr w9, [sp, #176]
-; NO_NEON-NEXT: ldr w10, [sp, #48]
-; NO_NEON-NEXT: str w8, [x0, #100]
-; NO_NEON-NEXT: ldr w8, [sp, #112]
-; NO_NEON-NEXT: str w10, [x0, #84]
-; NO_NEON-NEXT: ldr w10, [sp, #40]
-; NO_NEON-NEXT: stp w8, w9, [x0, #88]
+; NO_NEON-NEXT: str w5, [x0, #64]
+; NO_NEON-NEXT: stp w8, w9, [x0, #100]
+; NO_NEON-NEXT: ldr w8, [sp, #176]
+; NO_NEON-NEXT: ldr w9, [sp, #112]
+; NO_NEON-NEXT: str w4, [x0, #48]
+; NO_NEON-NEXT: str w8, [x0, #92]
+; NO_NEON-NEXT: ldr w8, [sp, #48]
+; NO_NEON-NEXT: str w3, [x0, #32]
+; NO_NEON-NEXT: stp w8, w9, [x0, #84]
; NO_NEON-NEXT: ldr w8, [sp, #168]
; NO_NEON-NEXT: ldr w9, [sp, #104]
-; NO_NEON-NEXT: str w10, [x0, #68]
-; NO_NEON-NEXT: ldr w10, [sp, #32]
-; NO_NEON-NEXT: stp w9, w8, [x0, #72]
+; NO_NEON-NEXT: str w2, [x0, #16]
+; NO_NEON-NEXT: str w8, [x0, #76]
+; NO_NEON-NEXT: ldr w8, [sp, #40]
+; NO_NEON-NEXT: str w1, [x0]
+; NO_NEON-NEXT: stp w8, w9, [x0, #68]
; NO_NEON-NEXT: ldr w8, [sp, #160]
; NO_NEON-NEXT: ldr w9, [sp, #96]
-; NO_NEON-NEXT: str w10, [x0, #52]
-; NO_NEON-NEXT: ldr w10, [sp, #24]
-; NO_NEON-NEXT: stp w9, w8, [x0, #56]
+; NO_NEON-NEXT: str w8, [x0, #60]
+; NO_NEON-NEXT: ldr w8, [sp, #32]
+; NO_NEON-NEXT: stp w8, w9, [x0, #52]
; NO_NEON-NEXT: ldr w8, [sp, #152]
; NO_NEON-NEXT: ldr w9, [sp, #88]
-; NO_NEON-NEXT: str w10, [x0, #36]
-; NO_NEON-NEXT: ldr w10, [sp, #16]
-; NO_NEON-NEXT: stp w9, w8, [x0, #40]
+; NO_NEON-NEXT: str w8, [x0, #44]
+; NO_NEON-NEXT: ldr w8, [sp, #24]
+; NO_NEON-NEXT: stp w8, w9, [x0, #36]
; NO_NEON-NEXT: ldr w8, [sp, #144]
; NO_NEON-NEXT: ldr w9, [sp, #80]
-; NO_NEON-NEXT: str w10, [x0, #20]
-; NO_NEON-NEXT: ldr w10, [sp, #8]
-; NO_NEON-NEXT: stp w9, w8, [x0, #24]
+; NO_NEON-NEXT: str w8, [x0, #28]
+; NO_NEON-NEXT: ldr w8, [sp, #16]
+; NO_NEON-NEXT: stp w8, w9, [x0, #20]
; NO_NEON-NEXT: ldr w8, [sp, #136]
; NO_NEON-NEXT: ldr w9, [sp, #72]
-; NO_NEON-NEXT: str w6, [x0, #80]
-; NO_NEON-NEXT: str w5, [x0, #64]
-; NO_NEON-NEXT: str w4, [x0, #48]
-; NO_NEON-NEXT: str w3, [x0, #32]
-; NO_NEON-NEXT: str w2, [x0, #16]
-; NO_NEON-NEXT: stp w9, w8, [x0, #8]
-; NO_NEON-NEXT: stp w1, w10, [x0]
+; NO_NEON-NEXT: str w8, [x0, #12]
+; NO_NEON-NEXT: ldr w8, [sp, #8]
+; NO_NEON-NEXT: stp w8, w9, [x0, #4]
; NO_NEON-NEXT: ret
%interleaved.intrinsic = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3)
store <32 x i32> %interleaved.intrinsic, ptr %ptr, align 4
@@ -2325,31 +3044,6 @@ define void @store_undef_mask_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i3
store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
}
-
-declare <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32>, <4 x i32>, <4 x i32>)
-
-declare <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32>, <8 x i32>)
-
-declare <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>)
-
-declare <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8>, <8 x i8>)
-
-declare <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32>, <8 x i32>, <8 x i32>)
-
-declare <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32>, <8 x i32>, <8 x i32>, <8 x i32>)
-
-declare <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr>, <2 x ptr>)
-
-declare <6 x ptr> @llvm.vector.interleave3.v6p0(<2 x ptr>, <2 x ptr>, <2 x ptr>)
-
-declare <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32>, <4 x i32>)
-
-declare <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>)
-
-declare <8 x ptr> @llvm.vector.interleave4.v8p0(<2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr>)
-
-declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64>)
-
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; NO_NEON-IADISABLED: {{.*}}
; NO_NEON-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index ff10464b1de89..30f944e0a050d 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1,41 +1,258 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-
-target triple = "aarch64-linux-gnu"
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
define void @load_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: load_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: uzp1 z2.h, z1.h, z0.h
+; CHECK-IADISABLED-NEXT: uzp2 z0.h, z1.h, z0.h
+; CHECK-IADISABLED-NEXT: // fake_use: $z2
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <32 x i16>, ptr %ptr, align 4
%v0 = shufflevector <32 x i16> %interleaved.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14,
i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%v1 = shufflevector <32 x i16> %interleaved.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15,
i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+ tail call void (...) @llvm.fake.use(<16 x i16> %v0)
+ tail call void (...) @llvm.fake.use(<16 x i16> %v1)
ret void
}
define void @load_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: load_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.s
+; CHECK-IAENABLED-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2 $z0_z1_z2
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: mov w11, v1.s[3]
+; CHECK-IADISABLED-NEXT: mov z3.s, z2.s[7]
+; CHECK-IADISABLED-NEXT: mov z4.s, z2.s[4]
+; CHECK-IADISABLED-NEXT: fmov w9, s1
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[6]
+; CHECK-IADISABLED-NEXT: mov w10, v2.s[1]
+; CHECK-IADISABLED-NEXT: fmov w12, s4
+; CHECK-IADISABLED-NEXT: stp w9, w11, [sp, #160]
+; CHECK-IADISABLED-NEXT: fmov w9, s3
+; CHECK-IADISABLED-NEXT: mov z3.s, z0.s[5]
+; CHECK-IADISABLED-NEXT: stp w12, w9, [sp, #176]
+; CHECK-IADISABLED-NEXT: fmov w12, s5
+; CHECK-IADISABLED-NEXT: mov w9, v0.s[2]
+; CHECK-IADISABLED-NEXT: stp w12, w10, [sp, #168]
+; CHECK-IADISABLED-NEXT: fmov w10, s3
+; CHECK-IADISABLED-NEXT: stp w9, w10, [sp, #88]
+; CHECK-IADISABLED-NEXT: ldr z3, [x8]
+; CHECK-IADISABLED-NEXT: mov w11, v3.s[1]
+; CHECK-IADISABLED-NEXT: mov z5.s, z3.s[4]
+; CHECK-IADISABLED-NEXT: fmov w9, s3
+; CHECK-IADISABLED-NEXT: mov w8, v3.s[3]
+; CHECK-IADISABLED-NEXT: mov w10, v3.s[2]
+; CHECK-IADISABLED-NEXT: mov z4.s, z3.s[5]
+; CHECK-IADISABLED-NEXT: mov z3.s, z2.s[5]
+; CHECK-IADISABLED-NEXT: stp w9, w11, [sp, #64]
+; CHECK-IADISABLED-NEXT: fmov w9, s5
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[4]
+; CHECK-IADISABLED-NEXT: str s4, [sp, #84]
+; CHECK-IADISABLED-NEXT: mov z4.s, z1.s[7]
+; CHECK-IADISABLED-NEXT: mov w11, v1.s[1]
+; CHECK-IADISABLED-NEXT: str w10, [sp, #72]
+; CHECK-IADISABLED-NEXT: fmov w10, s3
+; CHECK-IADISABLED-NEXT: stp w8, w9, [sp, #76]
+; CHECK-IADISABLED-NEXT: mov w9, v2.s[2]
+; CHECK-IADISABLED-NEXT: add x8, sp, #64
+; CHECK-IADISABLED-NEXT: ldr z3, [x8]
+; CHECK-IADISABLED-NEXT: fmov w8, s4
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[6]
+; CHECK-IADISABLED-NEXT: str w11, [sp, #128]
+; CHECK-IADISABLED-NEXT: mov w11, v1.s[2]
+; CHECK-IADISABLED-NEXT: stp w9, w10, [sp, #140]
+; CHECK-IADISABLED-NEXT: fmov w9, s5
+; CHECK-IADISABLED-NEXT: fmov w10, s0
+; CHECK-IADISABLED-NEXT: stp w9, w8, [sp, #132]
+; CHECK-IADISABLED-NEXT: mov w8, v0.s[3]
+; CHECK-IADISABLED-NEXT: fmov w9, s4
+; CHECK-IADISABLED-NEXT: str w10, [sp, #52]
+; CHECK-IADISABLED-NEXT: stp w8, w9, [sp, #56]
+; CHECK-IADISABLED-NEXT: add x8, sp, #128
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: mov w8, v4.s[3]
+; CHECK-IADISABLED-NEXT: mov w9, v4.s[2]
+; CHECK-IADISABLED-NEXT: mov w10, v4.s[1]
+; CHECK-IADISABLED-NEXT: mov z5.s, z4.s[4]
+; CHECK-IADISABLED-NEXT: stp w9, w8, [sp, #40]
+; CHECK-IADISABLED-NEXT: fmov w8, s4
+; CHECK-IADISABLED-NEXT: mov z4.s, z2.s[6]
+; CHECK-IADISABLED-NEXT: str s5, [sp, #48]
+; CHECK-IADISABLED-NEXT: mov w9, v2.s[3]
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[5]
+; CHECK-IADISABLED-NEXT: stp w8, w10, [sp, #32]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: fmov w10, s2
+; CHECK-IADISABLED-NEXT: ldr z1, [x8]
+; CHECK-IADISABLED-NEXT: str s4, [sp, #112]
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[4]
+; CHECK-IADISABLED-NEXT: mov w8, v0.s[1]
+; CHECK-IADISABLED-NEXT: mov z2.s, z0.s[7]
+; CHECK-IADISABLED-NEXT: str s5, [sp, #100]
+; CHECK-IADISABLED-NEXT: stp w10, w9, [sp, #104]
+; CHECK-IADISABLED-NEXT: fmov w9, s4
+; CHECK-IADISABLED-NEXT: str w11, [sp, #96]
+; CHECK-IADISABLED-NEXT: str s2, [sp, #28]
+; CHECK-IADISABLED-NEXT: stp w8, w9, [sp, #20]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov w8, v0.s[3]
+; CHECK-IADISABLED-NEXT: mov w9, v0.s[2]
+; CHECK-IADISABLED-NEXT: mov w10, v0.s[1]
+; CHECK-IADISABLED-NEXT: mov z2.s, z0.s[4]
+; CHECK-IADISABLED-NEXT: stp w9, w8, [sp, #8]
+; CHECK-IADISABLED-NEXT: fmov w8, s0
+; CHECK-IADISABLED-NEXT: str s2, [sp, #16]
+; CHECK-IADISABLED-NEXT: stp w8, w10, [sp]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z3
+; CHECK-IADISABLED-NEXT: // fake_use: $z1
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
%v2 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ tail call void (...) @llvm.fake.use(<8 x i32> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v1)
+ tail call void (...) @llvm.fake.use(<8 x i32> %v2)
ret void
}
define void @load_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: load_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z0_z1_z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z0, [x0]
+; CHECK-IADISABLED-NEXT: add x8, sp, #128
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z3, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: zip1 z4.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: trn2 z7.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: stp d3, d2, [sp, #144]
+; CHECK-IADISABLED-NEXT: mov z5.d, z4.d[1]
+; CHECK-IADISABLED-NEXT: fmov x9, d4
+; CHECK-IADISABLED-NEXT: str x9, [sp, #96]
+; CHECK-IADISABLED-NEXT: str d5, [sp, #104]
+; CHECK-IADISABLED-NEXT: zip1 z5.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: mov z16.d, z5.d[3]
+; CHECK-IADISABLED-NEXT: mov z5.d, z5.d[2]
+; CHECK-IADISABLED-NEXT: mov z6.d, z4.d[2]
+; CHECK-IADISABLED-NEXT: mov z4.d, z4.d[3]
+; CHECK-IADISABLED-NEXT: fmov x9, d6
+; CHECK-IADISABLED-NEXT: str d4, [sp, #120]
+; CHECK-IADISABLED-NEXT: mov z6.d, z7.d[1]
+; CHECK-IADISABLED-NEXT: str x9, [sp, #112]
+; CHECK-IADISABLED-NEXT: fmov x9, d7
+; CHECK-IADISABLED-NEXT: trn1 z7.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: str d16, [sp, #88]
+; CHECK-IADISABLED-NEXT: zip2 z16.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: fmov x8, d5
+; CHECK-IADISABLED-NEXT: trn2 z2.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: str d6, [sp, #72]
+; CHECK-IADISABLED-NEXT: mov z5.d, z7.d[2]
+; CHECK-IADISABLED-NEXT: str x9, [sp, #64]
+; CHECK-IADISABLED-NEXT: add x9, sp, #64
+; CHECK-IADISABLED-NEXT: mov z6.d, z16.d[1]
+; CHECK-IADISABLED-NEXT: mov z7.d, z7.d[3]
+; CHECK-IADISABLED-NEXT: mov z1.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #80]
+; CHECK-IADISABLED-NEXT: fmov x8, d16
+; CHECK-IADISABLED-NEXT: mov z3.d, z2.d[3]
+; CHECK-IADISABLED-NEXT: ldr z16, [x9]
+; CHECK-IADISABLED-NEXT: mov z2.d, z2.d[2]
+; CHECK-IADISABLED-NEXT: mov z0.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: str d7, [sp, #56]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #32]
+; CHECK-IADISABLED-NEXT: fmov x8, d5
+; CHECK-IADISABLED-NEXT: str d6, [sp, #40]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #48]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: ldr z5, [x8]
+; CHECK-IADISABLED-NEXT: fmov x8, d2
+; CHECK-IADISABLED-NEXT: stp d0, d1, [sp, #160]
+; CHECK-IADISABLED-NEXT: str d3, [sp, #24]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #16]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov z1.d, z0.d[1]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: str x8, [sp]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: str d1, [sp, #8]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z16
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <16 x i64>, ptr %ptr, align 4
%v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%v2 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%v3 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ tail call void (...) @llvm.fake.use(<4 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i64> %v1)
+ tail call void (...) @llvm.fake.use(<4 x i64> %v2)
+ tail call void (...) @llvm.fake.use(<4 x i64> %v3)
ret void
}
@@ -166,7 +383,7 @@ define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
- i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19,
i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
store <24 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -225,35 +442,218 @@ define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2
}
define void @load_ptrvec_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: load_ptrvec_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_ptrvec_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_ptrvec_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: uzp1 z2.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: // fake_use: $z2
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v1)
ret void
}
define void @load_ptrvec_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: load_ptrvec_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_ptrvec_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2 $z0_z1_z2
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_ptrvec_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: mov z3.d, z2.d[2]
+; CHECK-IADISABLED-NEXT: mov z4.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: fmov x11, d1
+; CHECK-IADISABLED-NEXT: mov x9, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov z5.d, z0.d[2]
+; CHECK-IADISABLED-NEXT: fmov x10, d3
+; CHECK-IADISABLED-NEXT: str x9, [sp, #88]
+; CHECK-IADISABLED-NEXT: str x10, [sp, #176]
+; CHECK-IADISABLED-NEXT: fmov x10, d4
+; CHECK-IADISABLED-NEXT: stp x11, x10, [sp, #160]
+; CHECK-IADISABLED-NEXT: add x10, sp, #64
+; CHECK-IADISABLED-NEXT: ldr z3, [x8]
+; CHECK-IADISABLED-NEXT: mov x8, v3.d[1]
+; CHECK-IADISABLED-NEXT: fmov x9, d3
+; CHECK-IADISABLED-NEXT: mov z4.d, z3.d[2]
+; CHECK-IADISABLED-NEXT: mov z3.d, z2.d[3]
+; CHECK-IADISABLED-NEXT: str d4, [sp, #80]
+; CHECK-IADISABLED-NEXT: stp x9, x8, [sp, #64]
+; CHECK-IADISABLED-NEXT: mov x9, v1.d[1]
+; CHECK-IADISABLED-NEXT: fmov x8, d2
+; CHECK-IADISABLED-NEXT: ldr z4, [x10]
+; CHECK-IADISABLED-NEXT: mov z2.d, z2.d[1]
+; CHECK-IADISABLED-NEXT: mov z1.d, z1.d[2]
+; CHECK-IADISABLED-NEXT: stp x9, x8, [sp, #128]
+; CHECK-IADISABLED-NEXT: fmov x8, d3
+; CHECK-IADISABLED-NEXT: add x9, sp, #128
+; CHECK-IADISABLED-NEXT: str x8, [sp, #144]
+; CHECK-IADISABLED-NEXT: fmov x8, d5
+; CHECK-IADISABLED-NEXT: str x8, [sp, #56]
+; CHECK-IADISABLED-NEXT: ldr z3, [x9]
+; CHECK-IADISABLED-NEXT: mov x8, v3.d[1]
+; CHECK-IADISABLED-NEXT: mov z5.d, z3.d[2]
+; CHECK-IADISABLED-NEXT: fmov x9, d3
+; CHECK-IADISABLED-NEXT: mov z3.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: str d5, [sp, #48]
+; CHECK-IADISABLED-NEXT: stp x9, x8, [sp, #32]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: ldr z5, [x8]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: stp d1, d2, [sp, #96]
+; CHECK-IADISABLED-NEXT: str d3, [sp, #24]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #16]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov z1.d, z0.d[1]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: str x8, [sp]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: str d1, [sp, #8]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <12 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
%v2 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v1)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v2)
ret void
}
define void @load_ptrvec_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: load_ptrvec_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_ptrvec_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z0_z1_z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_ptrvec_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z0, [x0]
+; CHECK-IADISABLED-NEXT: add x8, sp, #128
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z3, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: zip1 z4.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: trn2 z7.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: stp d3, d2, [sp, #144]
+; CHECK-IADISABLED-NEXT: mov z5.d, z4.d[1]
+; CHECK-IADISABLED-NEXT: fmov x9, d4
+; CHECK-IADISABLED-NEXT: str x9, [sp, #96]
+; CHECK-IADISABLED-NEXT: str d5, [sp, #104]
+; CHECK-IADISABLED-NEXT: zip1 z5.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: mov z16.d, z5.d[3]
+; CHECK-IADISABLED-NEXT: mov z5.d, z5.d[2]
+; CHECK-IADISABLED-NEXT: mov z6.d, z4.d[2]
+; CHECK-IADISABLED-NEXT: mov z4.d, z4.d[3]
+; CHECK-IADISABLED-NEXT: fmov x9, d6
+; CHECK-IADISABLED-NEXT: str d4, [sp, #120]
+; CHECK-IADISABLED-NEXT: mov z6.d, z7.d[1]
+; CHECK-IADISABLED-NEXT: str x9, [sp, #112]
+; CHECK-IADISABLED-NEXT: fmov x9, d7
+; CHECK-IADISABLED-NEXT: trn1 z7.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: str d16, [sp, #88]
+; CHECK-IADISABLED-NEXT: zip2 z16.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: fmov x8, d5
+; CHECK-IADISABLED-NEXT: trn2 z2.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: str d6, [sp, #72]
+; CHECK-IADISABLED-NEXT: mov z5.d, z7.d[2]
+; CHECK-IADISABLED-NEXT: str x9, [sp, #64]
+; CHECK-IADISABLED-NEXT: add x9, sp, #64
+; CHECK-IADISABLED-NEXT: mov z6.d, z16.d[1]
+; CHECK-IADISABLED-NEXT: mov z7.d, z7.d[3]
+; CHECK-IADISABLED-NEXT: mov z1.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #80]
+; CHECK-IADISABLED-NEXT: fmov x8, d16
+; CHECK-IADISABLED-NEXT: mov z3.d, z2.d[3]
+; CHECK-IADISABLED-NEXT: ldr z16, [x9]
+; CHECK-IADISABLED-NEXT: mov z2.d, z2.d[2]
+; CHECK-IADISABLED-NEXT: mov z0.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: str d7, [sp, #56]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #32]
+; CHECK-IADISABLED-NEXT: fmov x8, d5
+; CHECK-IADISABLED-NEXT: str d6, [sp, #40]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #48]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: ldr z5, [x8]
+; CHECK-IADISABLED-NEXT: fmov x8, d2
+; CHECK-IADISABLED-NEXT: stp d0, d1, [sp, #160]
+; CHECK-IADISABLED-NEXT: str d3, [sp, #24]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #16]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov z1.d, z0.d[1]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: str x8, [sp]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: str d1, [sp, #8]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z16
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <16 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%v2 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%v3 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v0)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v1)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v2)
+ tail call void (...) @llvm.fake.use(<4 x ptr> %v3)
ret void
}
@@ -344,7 +744,7 @@ define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-IADISABLED-NEXT: ret
%s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %s1 = shufflevector <4 x ptr> %v2, <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s1 = shufflevector <4 x ptr> %v2, <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
store <12 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -403,34 +803,59 @@ define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
}
define void @load_factor2_wide(ptr %ptr) #0 {
-; CHECK-LABEL: load_factor2_wide:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_factor2_wide:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: ld2d { z2.d, z3.d }, p0/z, [x0, #2, mul vl]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_factor2_wide:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z3, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: uzp1 z4.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp1 z5.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: uzp2 z2.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: // fake_use: $z2
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <16 x i64>, ptr %ptr, align 4
%v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ tail call void (...) @llvm.fake.use(<8 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<8 x i64> %v1)
ret void
}
define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
; CHECK-IAENABLED-LABEL: store_factor2_wide:
; CHECK-IAENABLED: // %bb.0:
-; CHECK-IAENABLED-NEXT: mov v17.16b, v4.16b
-; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
-; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: mov v17.16b, v6.16b
+; CHECK-IAENABLED-NEXT: mov v19.16b, v4.16b
; CHECK-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-IAENABLED-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-IAENABLED-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-IAENABLED-NEXT: mov v16.16b, v0.16b
+; CHECK-IAENABLED-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: mov v16.16b, v2.16b
+; CHECK-IAENABLED-NEXT: mov v18.16b, v0.16b
; CHECK-IAENABLED-NEXT: ptrue p1.d
-; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z5.d
-; CHECK-IAENABLED-NEXT: // kill: def $q6 killed $q6 def $z5_z6
-; CHECK-IAENABLED-NEXT: mov v5.16b, v2.16b
-; CHECK-IAENABLED-NEXT: splice z5.d, p0, z5.d, z3.d
-; CHECK-IAENABLED-NEXT: st2d { z16.d, z17.d }, p1, [x0]
-; CHECK-IAENABLED-NEXT: splice z6.d, p0, z6.d, z7.d
-; CHECK-IAENABLED-NEXT: st2d { z5.d, z6.d }, p1, [x0, #2, mul vl]
+; CHECK-IAENABLED-NEXT: splice z18.d, p0, z18.d, z1.d
+; CHECK-IAENABLED-NEXT: splice z16.d, p0, z16.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z19.d, p0, z19.d, z5.d
+; CHECK-IAENABLED-NEXT: splice z17.d, p0, z17.d, z7.d
+; CHECK-IAENABLED-NEXT: st2d { z18.d, z19.d }, p1, [x0]
+; CHECK-IAENABLED-NEXT: st2d { z16.d, z17.d }, p1, [x0, #2, mul vl]
; CHECK-IAENABLED-NEXT: ret
;
; CHECK-IADISABLED-LABEL: store_factor2_wide:
@@ -467,34 +892,123 @@ define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
; Check that neon is used for illegal multiples of 128-bit types
define void @load_384bit(ptr %ptr) #0 {
-; CHECK-LABEL: load_384bit:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_384bit:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: add x8, x0, #64
+; CHECK-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0], #32
+; CHECK-IAENABLED-NEXT: ld2 { v2.2d, v3.2d }, [x0]
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl2
+; CHECK-IAENABLED-NEXT: splice z1.d, p0, z1.d, z3.d
+; CHECK-IAENABLED-NEXT: splice z0.d, p0, z0.d, z2.d
+; CHECK-IAENABLED-NEXT: ld2 { v2.2d, v3.2d }, [x8]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_384bit:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: uzp1 z3.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp1 z4.d, z2.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z2.d, z2.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: // fake_use: $z3
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: // fake_use: $z2
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <12 x i64>, ptr %ptr, align 4
%v0 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10>
%v1 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
+ tail call void (...) @llvm.fake.use(<6 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<6 x i64> %v1)
ret void
}
; Check that neon is used for 128-bit vectors
define void @load_128bit(ptr %ptr) #0 {
-; CHECK-LABEL: load_128bit:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_128bit:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ld2 { v0.2d, v1.2d }, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $q0
+; CHECK-IAENABLED-NEXT: // fake_use: $q1 $q0_q1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_128bit:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0]
+; CHECK-IADISABLED-NEXT: movprfx z1, z0
+; CHECK-IADISABLED-NEXT: ext z1.b, z1.b, z0.b, #16
+; CHECK-IADISABLED-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-IADISABLED-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-IADISABLED-NEXT: // fake_use: $q2
+; CHECK-IADISABLED-NEXT: // fake_use: $q0
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <4 x i64>, ptr %ptr, align 4
%v0 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+ tail call void (...) @llvm.fake.use(<2 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<2 x i64> %v1)
ret void
}
; Check that correct ptrues are generated for min != max case
define void @load_min_not_max(ptr %ptr) #1 {
-; CHECK-LABEL: load_min_not_max:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_min_not_max:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_min_not_max:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #80
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IADISABLED-NEXT: mov x8, #4 // =0x4
+; CHECK-IADISABLED-NEXT: mov x10, sp
+; CHECK-IADISABLED-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; CHECK-IADISABLED-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: mov z3.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: mov z2.d, z0.d[2]
+; CHECK-IADISABLED-NEXT: mov z4.d, z1.d[2]
+; CHECK-IADISABLED-NEXT: fmov x9, d1
+; CHECK-IADISABLED-NEXT: str x8, [sp, #16]
+; CHECK-IADISABLED-NEXT: mov x8, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov z0.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: fmov x11, d3
+; CHECK-IADISABLED-NEXT: str x9, [sp]
+; CHECK-IADISABLED-NEXT: mov x9, v1.d[1]
+; CHECK-IADISABLED-NEXT: str d2, [sp, #24]
+; CHECK-IADISABLED-NEXT: str d4, [sp, #8]
+; CHECK-IADISABLED-NEXT: ld1d { z1.d }, p0/z, [x10]
+; CHECK-IADISABLED-NEXT: stp x8, x11, [sp, #48]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: stp x9, x8, [sp, #32]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: ld1d { z0.d }, p0/z, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z1
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <8 x i64>, ptr %ptr, align 4
%v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ tail call void (...) @llvm.fake.use(<4 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i64> %v1)
ret void
}
@@ -538,12 +1052,57 @@ define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
; Check that correct ptrues are generated for min > type case
define void @load_min_ge_type(ptr %ptr) #2 {
-; CHECK-LABEL: load_min_ge_type:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_min_ge_type:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_min_ge_type:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #80
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z0, [x0]
+; CHECK-IADISABLED-NEXT: ptrue p0.d, vl4
+; CHECK-IADISABLED-NEXT: movprfx z1, z0
+; CHECK-IADISABLED-NEXT: ext z1.b, z1.b, z0.b, #32
+; CHECK-IADISABLED-NEXT: mov z2.d, z0.d[2]
+; CHECK-IADISABLED-NEXT: fmov x8, d0
+; CHECK-IADISABLED-NEXT: mov x9, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov z0.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: mov z3.d, z1.d[2]
+; CHECK-IADISABLED-NEXT: fmov x10, d2
+; CHECK-IADISABLED-NEXT: mov z2.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: stp x8, x10, [sp, #32]
+; CHECK-IADISABLED-NEXT: fmov x8, d1
+; CHECK-IADISABLED-NEXT: fmov x10, d3
+; CHECK-IADISABLED-NEXT: stp x8, x10, [sp, #48]
+; CHECK-IADISABLED-NEXT: mov x8, v1.d[1]
+; CHECK-IADISABLED-NEXT: add x10, sp, #32
+; CHECK-IADISABLED-NEXT: ld1d { z1.d }, p0/z, [x10]
+; CHECK-IADISABLED-NEXT: str x9, [sp]
+; CHECK-IADISABLED-NEXT: str d0, [sp, #8]
+; CHECK-IADISABLED-NEXT: str d2, [sp, #24]
+; CHECK-IADISABLED-NEXT: str x8, [sp, #16]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: ld1d { z0.d }, p0/z, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z1
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <8 x i64>, ptr %ptr, align 4
%v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ tail call void (...) @llvm.fake.use(<4 x i64> %v0)
+ tail call void (...) @llvm.fake.use(<4 x i64> %v1)
ret void
}
@@ -579,45 +1138,256 @@ define void @store_min_ge_type(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
}
define void @load_double_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: load_double_factor4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_double_factor4:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z0_z1_z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_double_factor4:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z0, [x0]
+; CHECK-IADISABLED-NEXT: add x8, sp, #128
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z3, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: zip1 z4.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: trn2 z7.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: stp d3, d2, [sp, #144]
+; CHECK-IADISABLED-NEXT: mov z5.d, z4.d[1]
+; CHECK-IADISABLED-NEXT: stp d4, d5, [sp, #96]
+; CHECK-IADISABLED-NEXT: zip1 z5.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: mov z16.d, z5.d[3]
+; CHECK-IADISABLED-NEXT: mov z5.d, z5.d[2]
+; CHECK-IADISABLED-NEXT: mov z6.d, z4.d[3]
+; CHECK-IADISABLED-NEXT: mov z4.d, z4.d[2]
+; CHECK-IADISABLED-NEXT: stp d4, d6, [sp, #112]
+; CHECK-IADISABLED-NEXT: mov z4.d, z7.d[1]
+; CHECK-IADISABLED-NEXT: trn1 z6.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: ldr z17, [x8]
+; CHECK-IADISABLED-NEXT: stp d5, d16, [sp, #80]
+; CHECK-IADISABLED-NEXT: trn2 z2.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: add x8, sp, #64
+; CHECK-IADISABLED-NEXT: stp d7, d4, [sp, #64]
+; CHECK-IADISABLED-NEXT: zip2 z7.d, z0.d, z1.d
+; CHECK-IADISABLED-NEXT: mov z5.d, z6.d[3]
+; CHECK-IADISABLED-NEXT: mov z3.d, z6.d[2]
+; CHECK-IADISABLED-NEXT: ldr z6, [x8]
+; CHECK-IADISABLED-NEXT: mov z1.d, z1.d[3]
+; CHECK-IADISABLED-NEXT: mov z0.d, z0.d[3]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: mov z4.d, z7.d[1]
+; CHECK-IADISABLED-NEXT: stp d3, d5, [sp, #48]
+; CHECK-IADISABLED-NEXT: mov z3.d, z2.d[3]
+; CHECK-IADISABLED-NEXT: mov z2.d, z2.d[2]
+; CHECK-IADISABLED-NEXT: stp d7, d4, [sp, #32]
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: stp d0, d1, [sp, #160]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: stp d2, d3, [sp, #16]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: mov z1.d, z0.d[1]
+; CHECK-IADISABLED-NEXT: stp d0, d1, [sp]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z17
+; CHECK-IADISABLED-NEXT: // fake_use: $z6
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <16 x double>, ptr %ptr, align 4
%v0 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%v2 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%v3 = shufflevector <16 x double> %interleaved.vec, <16 x double> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+ tail call void (...) @llvm.fake.use(<4 x double> %v0)
+ tail call void (...) @llvm.fake.use(<4 x double> %v1)
+ tail call void (...) @llvm.fake.use(<4 x double> %v2)
+ tail call void (...) @llvm.fake.use(<4 x double> %v3)
ret void
}
define void @load_float_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: load_float_factor3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_float_factor3:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.s
+; CHECK-IAENABLED-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z2 $z0_z1_z2
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_float_factor3:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-IADISABLED-NEXT: sub x9, sp, #208
+; CHECK-IADISABLED-NEXT: mov x29, sp
+; CHECK-IADISABLED-NEXT: and sp, x9, #0xffffffffffffffe0
+; CHECK-IADISABLED-NEXT: .cfi_def_cfa w29, 16
+; CHECK-IADISABLED-NEXT: .cfi_offset w30, -8
+; CHECK-IADISABLED-NEXT: .cfi_offset w29, -16
+; CHECK-IADISABLED-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z2, [x0]
+; CHECK-IADISABLED-NEXT: add x8, sp, #160
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: mov z3.s, z1.s[7]
+; CHECK-IADISABLED-NEXT: mov z4.s, z1.s[4]
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[1]
+; CHECK-IADISABLED-NEXT: mov z6.s, z2.s[6]
+; CHECK-IADISABLED-NEXT: stp s4, s3, [sp, #176]
+; CHECK-IADISABLED-NEXT: mov z3.s, z2.s[3]
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[5]
+; CHECK-IADISABLED-NEXT: stp s6, s5, [sp, #168]
+; CHECK-IADISABLED-NEXT: mov z5.s, z0.s[2]
+; CHECK-IADISABLED-NEXT: stp s2, s3, [sp, #160]
+; CHECK-IADISABLED-NEXT: stp s5, s4, [sp, #88]
+; CHECK-IADISABLED-NEXT: ldr z3, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #64
+; CHECK-IADISABLED-NEXT: mov z6.s, z3.s[1]
+; CHECK-IADISABLED-NEXT: mov z4.s, z3.s[5]
+; CHECK-IADISABLED-NEXT: mov z5.s, z3.s[4]
+; CHECK-IADISABLED-NEXT: mov z7.s, z3.s[3]
+; CHECK-IADISABLED-NEXT: stp s3, s6, [sp, #64]
+; CHECK-IADISABLED-NEXT: mov z3.s, z3.s[2]
+; CHECK-IADISABLED-NEXT: mov z6.s, z2.s[4]
+; CHECK-IADISABLED-NEXT: stp s5, s4, [sp, #80]
+; CHECK-IADISABLED-NEXT: mov z4.s, z1.s[5]
+; CHECK-IADISABLED-NEXT: mov z5.s, z1.s[2]
+; CHECK-IADISABLED-NEXT: stp s3, s7, [sp, #72]
+; CHECK-IADISABLED-NEXT: mov z7.s, z2.s[7]
+; CHECK-IADISABLED-NEXT: ldr z3, [x8]
+; CHECK-IADISABLED-NEXT: stp s5, s4, [sp, #140]
+; CHECK-IADISABLED-NEXT: mov z4.s, z2.s[1]
+; CHECK-IADISABLED-NEXT: mov z5.s, z0.s[6]
+; CHECK-IADISABLED-NEXT: str s0, [sp, #52]
+; CHECK-IADISABLED-NEXT: add x8, sp, #128
+; CHECK-IADISABLED-NEXT: stp s6, s7, [sp, #132]
+; CHECK-IADISABLED-NEXT: str s4, [sp, #128]
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[3]
+; CHECK-IADISABLED-NEXT: stp s4, s5, [sp, #56]
+; CHECK-IADISABLED-NEXT: ldr z4, [x8]
+; CHECK-IADISABLED-NEXT: add x8, sp, #32
+; CHECK-IADISABLED-NEXT: mov z5.s, z4.s[4]
+; CHECK-IADISABLED-NEXT: mov z6.s, z4.s[3]
+; CHECK-IADISABLED-NEXT: mov z7.s, z4.s[2]
+; CHECK-IADISABLED-NEXT: str s4, [sp, #32]
+; CHECK-IADISABLED-NEXT: stp s6, s5, [sp, #44]
+; CHECK-IADISABLED-NEXT: mov z5.s, z4.s[1]
+; CHECK-IADISABLED-NEXT: mov z6.s, z1.s[6]
+; CHECK-IADISABLED-NEXT: mov z4.s, z1.s[3]
+; CHECK-IADISABLED-NEXT: stp s5, s7, [sp, #36]
+; CHECK-IADISABLED-NEXT: ldr z5, [x8]
+; CHECK-IADISABLED-NEXT: stp s4, s6, [sp, #108]
+; CHECK-IADISABLED-NEXT: mov z6.s, z2.s[5]
+; CHECK-IADISABLED-NEXT: mov z2.s, z2.s[2]
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[4]
+; CHECK-IADISABLED-NEXT: add x8, sp, #96
+; CHECK-IADISABLED-NEXT: stp s6, s1, [sp, #100]
+; CHECK-IADISABLED-NEXT: mov z1.s, z0.s[7]
+; CHECK-IADISABLED-NEXT: mov z0.s, z0.s[1]
+; CHECK-IADISABLED-NEXT: str s2, [sp, #96]
+; CHECK-IADISABLED-NEXT: stp s4, s1, [sp, #24]
+; CHECK-IADISABLED-NEXT: str s0, [sp, #20]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: mov x8, sp
+; CHECK-IADISABLED-NEXT: mov z4.s, z0.s[1]
+; CHECK-IADISABLED-NEXT: mov z1.s, z0.s[4]
+; CHECK-IADISABLED-NEXT: mov z2.s, z0.s[3]
+; CHECK-IADISABLED-NEXT: stp s0, s4, [sp]
+; CHECK-IADISABLED-NEXT: mov z0.s, z0.s[2]
+; CHECK-IADISABLED-NEXT: stp s2, s1, [sp, #12]
+; CHECK-IADISABLED-NEXT: str s0, [sp, #8]
+; CHECK-IADISABLED-NEXT: ldr z0, [x8]
+; CHECK-IADISABLED-NEXT: // fake_use: $z3
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: mov sp, x29
+; CHECK-IADISABLED-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <24 x float>, ptr %ptr, align 4
%v0 = shufflevector <24 x float> %interleaved.vec, <24 x float> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x float> %interleaved.vec, <24 x float> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
%v2 = shufflevector <24 x float> %interleaved.vec, <24 x float> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+ tail call void (...) @llvm.fake.use(<8 x float> %v0)
+ tail call void (...) @llvm.fake.use(<8 x float> %v1)
+ tail call void (...) @llvm.fake.use(<8 x float> %v2)
ret void
}
define void @load_half_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: load_half_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_half_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_half_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: uzp1 z2.h, z1.h, z0.h
+; CHECK-IADISABLED-NEXT: uzp2 z0.h, z1.h, z0.h
+; CHECK-IADISABLED-NEXT: // fake_use: $z2
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <32 x half>, ptr %ptr, align 4
%v0 = shufflevector <32 x half> %interleaved.vec, <32 x half> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%v1 = shufflevector <32 x half> %interleaved.vec, <32 x half> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+ tail call void (...) @llvm.fake.use(<16 x half> %v0)
+ tail call void (...) @llvm.fake.use(<16 x half> %v1)
ret void
}
define void @load_bfloat_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: load_bfloat_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: load_bfloat_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.h
+; CHECK-IAENABLED-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: movprfx z2, z1
+; CHECK-IAENABLED-NEXT: ext z2.b, z2.b, z1.b, #16
+; CHECK-IAENABLED-NEXT: movprfx z3, z0
+; CHECK-IAENABLED-NEXT: ext z3.b, z3.b, z0.b, #16
+; CHECK-IAENABLED-NEXT: // fake_use: $q0
+; CHECK-IAENABLED-NEXT: // fake_use: $q3 $z3
+; CHECK-IAENABLED-NEXT: // fake_use: $q1 $z0_z1
+; CHECK-IAENABLED-NEXT: // fake_use: $q2 $z2
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: load_bfloat_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldp q1, q0, [x0]
+; CHECK-IADISABLED-NEXT: ldp q3, q2, [x0, #32]
+; CHECK-IADISABLED-NEXT: uzp1 v4.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT: uzp2 v0.8h, v1.8h, v0.8h
+; CHECK-IADISABLED-NEXT: uzp1 v5.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT: uzp2 v2.8h, v3.8h, v2.8h
+; CHECK-IADISABLED-NEXT: // fake_use: $q4
+; CHECK-IADISABLED-NEXT: // fake_use: $q5
+; CHECK-IADISABLED-NEXT: // fake_use: $q0
+; CHECK-IADISABLED-NEXT: // fake_use: $q2
+; CHECK-IADISABLED-NEXT: ret
%interleaved.vec = load <32 x bfloat>, ptr %ptr, align 4
%v0 = shufflevector <32 x bfloat> %interleaved.vec, <32 x bfloat> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%v1 = shufflevector <32 x bfloat> %interleaved.vec, <32 x bfloat> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+ tail call void (...) @llvm.fake.use(<16 x bfloat> %v0)
+ tail call void (...) @llvm.fake.use(<16 x bfloat> %v1)
ret void
}
@@ -762,7 +1532,7 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
%s0 = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x float> %v2, <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
- i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <16 x float> %s0, <16 x float> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19,
i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
store <24 x float> %interleaved.vec, ptr %ptr, align 4
@@ -846,13 +1616,38 @@ define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1
; Ensure vscale_range property does not affect scalable vector types.
define void @deinterleave_nxptr_factor2(ptr %ptr) #2 {
-; CHECK-LABEL: deinterleave_nxptr_factor2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ret
+; CHECK-IAENABLED-LABEL: deinterleave_nxptr_factor2:
+; CHECK-IAENABLED: // %bb.0:
+; CHECK-IAENABLED-NEXT: ptrue p0.d
+; CHECK-IAENABLED-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-IAENABLED-NEXT: ld2d { z2.d, z3.d }, p0/z, [x0, #2, mul vl]
+; CHECK-IAENABLED-NEXT: // fake_use: $z0
+; CHECK-IAENABLED-NEXT: // fake_use: $z2
+; CHECK-IAENABLED-NEXT: // fake_use: $z1 $z0_z1
+; CHECK-IAENABLED-NEXT: // fake_use: $z3 $z2_z3
+; CHECK-IAENABLED-NEXT: ret
+;
+; CHECK-IADISABLED-LABEL: deinterleave_nxptr_factor2:
+; CHECK-IADISABLED: // %bb.0:
+; CHECK-IADISABLED-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z1, [x0]
+; CHECK-IADISABLED-NEXT: ldr z2, [x0, #3, mul vl]
+; CHECK-IADISABLED-NEXT: ldr z3, [x0, #2, mul vl]
+; CHECK-IADISABLED-NEXT: uzp1 z4.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z0.d, z1.d, z0.d
+; CHECK-IADISABLED-NEXT: uzp2 z5.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: uzp1 z1.d, z3.d, z2.d
+; CHECK-IADISABLED-NEXT: // fake_use: $z4
+; CHECK-IADISABLED-NEXT: // fake_use: $z1
+; CHECK-IADISABLED-NEXT: // fake_use: $z0
+; CHECK-IADISABLED-NEXT: // fake_use: $z5
+; CHECK-IADISABLED-NEXT: ret
%wide.vec = load <vscale x 8 x double>, ptr %ptr, align 8
%ldN = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec)
%extract1 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %ldN, 0
%extract2 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %ldN, 1
+ tail call void (...) @llvm.fake.use(<vscale x 4 x double> %extract1)
+ tail call void (...) @llvm.fake.use(<vscale x 4 x double> %extract2)
ret void
}
More information about the llvm-commits
mailing list