[llvm] d5a2395 - [AArch64][GlobalISel] Add deinterleave load and store test coverage. NFC (#228419)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 06:20:47 PDT 2026
Author: David Green
Date: 2026-10-02T13:20:30Z
New Revision: d5a23954543f23c05b66ce70362da1a6f8491761
URL: https://github.com/llvm/llvm-project/commit/d5a23954543f23c05b66ce70362da1a6f8491761
DIFF: https://github.com/llvm/llvm-project/commit/d5a23954543f23c05b66ce70362da1a6f8491761.diff
LOG: [AArch64][GlobalISel] Add deinterleave load and store test coverage. NFC (#228419)
Correct some NFC comments whilst here too.
Added:
Modified:
llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
llvm/test/CodeGen/AArch64/binopshuffles.ll
llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
llvm/test/CodeGen/AArch64/interleaved-accesses.ll
llvm/test/CodeGen/AArch64/vldn_shuffle.ll
llvm/test/CodeGen/AArch64/zext-shuffle.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 93fcebf2a6dc7..42f13bd114b26 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -6470,7 +6470,7 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
Opc = AArch64::LD2i64;
else
- llvm_unreachable("Unexpected type for st2lane!");
+ llvm_unreachable("Unexpected type for ld2lane!");
if (!selectVectorLoadLaneIntrinsic(Opc, 2, I))
return false;
break;
@@ -6536,7 +6536,7 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
Opc = AArch64::LD3i64;
else
- llvm_unreachable("Unexpected type for st3lane!");
+ llvm_unreachable("Unexpected type for ld3lane!");
if (!selectVectorLoadLaneIntrinsic(Opc, 3, I))
return false;
break;
@@ -6602,7 +6602,7 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
Ty == LLT::fixed_vector(2, P0) || Ty == S64 || Ty == P0)
Opc = AArch64::LD4i64;
else
- llvm_unreachable("Unexpected type for st4lane!");
+ llvm_unreachable("Unexpected type for ld4lane!");
if (!selectVectorLoadLaneIntrinsic(Opc, 4, I))
return false;
break;
diff --git a/llvm/test/CodeGen/AArch64/binopshuffles.ll b/llvm/test/CodeGen/AArch64/binopshuffles.ll
index 0f2fe1cabe2b6..022322ac08f73 100644
--- a/llvm/test/CodeGen/AArch64/binopshuffles.ll
+++ b/llvm/test/CodeGen/AArch64/binopshuffles.ll
@@ -1,6 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for vld3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vld4_intrinsic
define <4 x float> @vld2(ptr %pSrc) {
; CHECK-IAENABLED-LABEL: vld2:
@@ -17,6 +21,14 @@ define <4 x float> @vld2(ptr %pSrc) {
; CHECK-IADISABLED-NEXT: fmul v0.4s, v0.4s, v0.4s
; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -56,6 +68,29 @@ define <4 x float> @vld3(ptr %pSrc) {
; CHECK-IADISABLED-NEXT: mov v1.s[3], v4.s[3]
; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldr q4, [x0, #32]
+; CHECK-GI-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov s0, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-GI-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[3]
+; CHECK-GI-NEXT: fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <12 x float>, ptr %pSrc, align 4
%l2 = fmul fast <12 x float> %wide.vec, %wide.vec
@@ -94,6 +129,25 @@ define <4 x float> @vld4(ptr %pSrc) {
; CHECK-IADISABLED-NEXT: mov v0.d[1], v2.d[1]
; CHECK-IADISABLED-NEXT: fadd v0.4s, v0.4s, v1.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q3, q4, [x0, #32]
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmul v3.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: mov s2, v0.s[2]
+; CHECK-GI-NEXT: mov s0, v0.s[3]
+; CHECK-GI-NEXT: mov v2.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
+; CHECK-GI-NEXT: fmul v1.4s, v4.4s, v4.4s
+; CHECK-GI-NEXT: mov v2.s[2], v3.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[3]
+; CHECK-GI-NEXT: mov v2.s[3], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v1.s[3]
+; CHECK-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <16 x float>, ptr %pSrc, align 4
%l3 = fmul fast <16 x float> %wide.vec, %wide.vec
@@ -126,6 +180,15 @@ define <4 x float> @twosrc(ptr %pSrc1, ptr %pSrc2) {
; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v0.4s
; CHECK-IADISABLED-NEXT: faddp v0.4s, v0.4s, v1.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x1]
+; CHECK-GI-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -157,6 +220,18 @@ define <4 x float> @twosrc2(ptr %pSrc1, ptr %pSrc2) {
; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v5.4s, v0.4s
; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v3.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc2:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q1, q2, [x0]
+; CHECK-GI-NEXT: ldp q3, q4, [x1]
+; CHECK-GI-NEXT: fmul v0.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: fmul v5.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT: uzp2 v3.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT: uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT: fmla v0.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc1, align 4
%wide.vec26 = load <8 x float>, ptr %pSrc2, align 4
@@ -176,6 +251,13 @@ define void @noncanonical(ptr %p0, ptr %p1, ptr %p2) {
; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-NEXT: str s0, [x2]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: noncanonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr d0, [x1]
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: str s0, [x2]
+; CHECK-GI-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = add <8 x i8> %v0, <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
@@ -189,6 +271,15 @@ define void @noncanonical2(ptr %p0, ptr %p1, ptr %p2) {
; CHECK-LABEL: noncanonical2:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: noncanonical2:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr d0, [x0]
+; CHECK-GI-NEXT: ldr d1, [x1]
+; CHECK-GI-NEXT: add v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: str s0, [x2]
+; CHECK-GI-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = load <8 x i8>, ptr %p1
@@ -206,6 +297,14 @@ define <4 x float> @noncanonical3(ptr %pSrc) {
; CHECK-NEXT: fmul v0.4s, v0.4s, v0.4s
; CHECK-NEXT: faddp v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: noncanonical3:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%wide.vec = load <8 x float>, ptr %pSrc, align 4
%l2 = fmul fast <8 x float> %wide.vec, %wide.vec
@@ -227,6 +326,17 @@ define void @noncanonical_extmask(ptr %p0, ptr %p1, ptr %p2) {
; CHECK-NEXT: uzp1 v0.8b, v1.8b, v0.8b
; CHECK-NEXT: str d0, [x2]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: noncanonical_extmask:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: adrp x8, .LCPI8_0
+; CHECK-GI-NEXT: ldr d1, [x0]
+; CHECK-GI-NEXT: ldr d0, [x8, :lo12:.LCPI8_0]
+; CHECK-GI-NEXT: add v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT: ldr d1, [x1]
+; CHECK-GI-NEXT: uzp1 v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT: str d0, [x2]
+; CHECK-GI-NEXT: ret
entry:
%v0 = load <8 x i8>, ptr %p0
%v1 = add <8 x i8> %v0, <i8 0, i8 1, i8 2, i8 3, i8 7, i8 7, i8 7, i8 7>
@@ -240,6 +350,10 @@ define void @skip_optimizing_dead_binop(ptr %p0, ptr %p1) {
; CHECK-LABEL: skip_optimizing_dead_binop:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: skip_optimizing_dead_binop:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ret
entry:
%v0 = load <8 x double>, ptr %p0
%shuffled_1 = shufflevector <8 x double> %v0, <8 x double> poison, <2 x i32> <i32 0, i32 4>
@@ -270,6 +384,18 @@ define <4 x float> @twosrc_intrinsic(ptr %pSrc1, ptr %pSrc2) {
; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc_intrinsic:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x1]
+; CHECK-GI-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v5.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: uzp2 v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: fmla v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
@@ -305,6 +431,18 @@ define <4 x float> @twosrc2_intrinsic(ptr %pSrc1, ptr %pSrc2) {
; CHECK-IADISABLED-NEXT: fmul v0.4s, v5.4s, v4.4s
; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v2.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc2_intrinsic:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x1]
+; CHECK-GI-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v5.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: uzp2 v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: fmul v0.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: fmla v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc1, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
@@ -336,6 +474,15 @@ define <4 x float> @vld2_intrinsic(ptr %pSrc) {
; CHECK-IADISABLED-NEXT: fmul v0.4s, v2.4s, v2.4s
; CHECK-IADISABLED-NEXT: fmla v0.4s, v1.4s, v1.4s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2_intrinsic:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: fmul v0.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v0.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%intrinsic.load.0 = load <8 x float>, ptr %pSrc, align 4
%ldN = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %intrinsic.load.0)
@@ -357,6 +504,14 @@ define <4 x float> @vld3_intrinsic(ptr %pSrc) {
; CHECK-NEXT: fmla v0.4s, v2.4s, v2.4s
; CHECK-NEXT: fmla v0.4s, v3.4s, v3.4s
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3_intrinsic:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ld3 { v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-GI-NEXT: fmul v0.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmla v0.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v0.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: ret
entry:
%intrinsic.load.0 = load <12 x float>, ptr %pSrc, align 4
%ldN = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %intrinsic.load.0)
@@ -381,6 +536,13 @@ define <4 x float> @vld4_intrinsic(ptr %pSrc) {
; CHECK-NEXT: fmul v0.4s, v3.4s, v3.4s
; CHECK-NEXT: fmla v0.4s, v4.4s, v4.4s
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4_intrinsic:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-GI-NEXT: fmul v0.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: fmla v0.4s, v4.4s, v4.4s
+; CHECK-GI-NEXT: ret
entry:
%intrinsic.load.0 = load <16 x float>, ptr %pSrc, align 4
%ldN = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %intrinsic.load.0)
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index 426fb4b472cbe..cf6e40839cc0d 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -1,8 +1,41 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for vector_deinterleave4_v8i8_v32i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v16i8_v64i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v4i16_v16i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v8i16_v32i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v2i32_v8i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v4i32_v16i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v1i64_v4i64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v2i64_v8i64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v4f16_v16f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v8f16_v32f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v2f32_v8f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v4f32_v16f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v1f64_v4f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v2f64_v8f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v4bf16_v16bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave4_v8bf16_v32bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v6f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v12f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v24f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v48i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v12f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vector_deinterleave3_v24i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v12f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v24i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v12i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v48bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v24i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave6_v96i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave8_v16f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave8_v32i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave8_v16i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for deinterleave8_v64bf16
define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) {
; CHECK-SD-LABEL: vector_deinterleave_v2f16_v4f16:
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
index ab06111fb52ff..c2866aa2ffce6 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
@@ -1,8 +1,40 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for interleave4_v32i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v64i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v16i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v32i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v8i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v16i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v4i64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v8i64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v16f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v32f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v8f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v16f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v4f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v8f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v16bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave4_v32bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v6f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v12f32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v24i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v24f16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v24bf16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v48i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v12i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave3_v24i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave6_v12f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave6_v24i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave6_v12i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave6_v24i16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave8_v16f64
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave8_v16i32
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for interleave8_v32i16
define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) {
; CHECK-LABEL: interleave2_v4f16:
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index 2e52b3b373484..f35d6a4679ddc 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -3,6 +3,31 @@
; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=NEON,NEON-IADISABLED
; RUN: llc -mtriple=aarch64-linux-gnu < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IAENABLED
; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s -mattr=-neon | FileCheck %s --check-prefixes=NO_NEON,NO_NEON-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for load_factor2_wide2
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_factor2_wide3
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_factor3_wide
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_factor4_wide
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_factor2_fp128
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_large_vector
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for load_large_vector_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor3_wide_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor4_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor4_wide_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor3_undeflane_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor3_undefmultimid_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor4_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor4_undefbeg_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor4_undefend_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor4_undefmid_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_general_mask_factor4_undefmulti_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_ptrvec_factor3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_ptrvec_factor4_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_undef_mask_factor3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_undef_mask_factor4_intrinsic
define void @load_factor2(ptr %ptr) {
; NEON-IAENABLED-LABEL: load_factor2:
@@ -56,6 +81,15 @@ define void @load_factor2(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w9
; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: uzp1 v1.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: // fake_use: $d1 $q1
+; CHECK-GI-NEXT: // fake_use: $d0 $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <16 x i8>, ptr %ptr, align 4
%v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -113,6 +147,27 @@ define void @load_factor3(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w12
; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov s4, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[1]
+; CHECK-GI-NEXT: ldr q0, [x0, #32]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v4.s[2], v0.s[0]
+; CHECK-GI-NEXT: mov v2.s[3], v0.s[1]
+; CHECK-GI-NEXT: mov v3.s[3], v0.s[2]
+; CHECK-GI-NEXT: mov v4.s[3], v0.s[3]
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -179,6 +234,28 @@ define void @load_factor4(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w9
; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov s0, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[2]
+; CHECK-GI-NEXT: ldp q1, q4, [x0, #32]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[1]
+; CHECK-GI-NEXT: mov v0.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v2.s[3], v4.s[0]
+; CHECK-GI-NEXT: mov v3.s[3], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[2]
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -235,6 +312,14 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
; NO_NEON-NEXT: strb w9, [x0, #1]
; NO_NEON-NEXT: strb w1, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: str q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i8> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -289,6 +374,24 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
; NO_NEON-NEXT: stp x8, x12, [x0, #16]
; NO_NEON-NEXT: stp x10, x9, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s3, v0.s[0]
+; CHECK-GI-NEXT: mov s4, v1.s[1]
+; CHECK-GI-NEXT: mov s5, v2.s[2]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v4.s[1], v2.s[1]
+; CHECK-GI-NEXT: mov v5.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[2], v2.s[0]
+; CHECK-GI-NEXT: mov v4.s[2], v0.s[2]
+; CHECK-GI-NEXT: mov v5.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v3.s[3], v0.s[1]
+; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
+; CHECK-GI-NEXT: mov v5.s[3], v2.s[3]
+; CHECK-GI-NEXT: stp q3, q4, [x0]
+; CHECK-GI-NEXT: str q5, [x0, #32]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -348,6 +451,28 @@ define void @store_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
; NO_NEON-NEXT: stp w5, w9, [x0, #4]
; NO_NEON-NEXT: str w1, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s4, v0.s[0]
+; CHECK-GI-NEXT: mov s5, v0.s[1]
+; CHECK-GI-NEXT: mov s6, v0.s[2]
+; CHECK-GI-NEXT: mov s0, v0.s[3]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v6.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v4.s[2], v2.s[0]
+; CHECK-GI-NEXT: mov v5.s[2], v2.s[1]
+; CHECK-GI-NEXT: mov v6.s[2], v2.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v4.s[3], v3.s[0]
+; CHECK-GI-NEXT: mov v5.s[3], v3.s[1]
+; CHECK-GI-NEXT: mov v6.s[3], v3.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v3.s[3]
+; CHECK-GI-NEXT: stp q4, q5, [x0]
+; CHECK-GI-NEXT: stp q6, q0, [x0, #32]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -381,6 +506,15 @@ define void @load_ptrvec_factor2(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $x9
; NO_NEON-NEXT: // fake_use: $x10
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_ptrvec_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <4 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x ptr> %interleaved.vec, <4 x ptr> poison, <2 x i32> <i32 1, i32 3>
@@ -422,6 +556,18 @@ define void @load_ptrvec_factor3(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $x12
; NO_NEON-NEXT: // fake_use: $x10
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_ptrvec_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldr q2, [x0, #32]
+; CHECK-GI-NEXT: ext v3.16b, v0.16b, v2.16b, #8
+; CHECK-GI-NEXT: mov v0.d[1], v1.d[1]
+; CHECK-GI-NEXT: mov v1.d[1], v2.d[1]
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q1
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <6 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 0, i32 3>
%v1 = shufflevector <6 x ptr> %interleaved.vec, <6 x ptr> poison, <2 x i32> <i32 1, i32 4>
@@ -471,6 +617,21 @@ define void @load_ptrvec_factor4(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $x12
; NO_NEON-NEXT: // fake_use: $x8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_ptrvec_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q2, q0, [x0, #16]
+; CHECK-GI-NEXT: ldr q1, [x0]
+; CHECK-GI-NEXT: ldr q3, [x0, #48]
+; CHECK-GI-NEXT: zip1 v4.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT: zip2 v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT: zip1 v1.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT: zip2 v2.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q1
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
%v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 0, i32 4>
%v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <2 x i32> <i32 1, i32 5>
@@ -503,6 +664,13 @@ define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
; NO_NEON-NEXT: stp x3, x5, [x0, #16]
; NO_NEON-NEXT: stp x2, x4, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
store <4 x ptr> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -532,6 +700,21 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
; NO_NEON-NEXT: stp x6, x3, [x0, #16]
; NO_NEON-NEXT: stp x2, x4, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v3.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v1.2d, v2.2d
+; CHECK-GI-NEXT: mov v2.d[1], v0.d[1]
+; CHECK-GI-NEXT: mov x8, v3.d[1]
+; CHECK-GI-NEXT: mov x9, v2.d[1]
+; CHECK-GI-NEXT: mov x10, v1.d[1]
+; CHECK-GI-NEXT: mov v3.d[1], x8
+; CHECK-GI-NEXT: mov v2.d[1], x9
+; CHECK-GI-NEXT: mov v1.d[1], x10
+; CHECK-GI-NEXT: stp q3, q2, [x0]
+; CHECK-GI-NEXT: str q1, [x0, #32]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
%interleaved.vec = shufflevector <4 x ptr> %s0, <4 x ptr> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
@@ -568,6 +751,16 @@ define void @store_ptrvec_factor4(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
; NO_NEON-NEXT: ldr x8, [sp]
; NO_NEON-NEXT: stp x6, x8, [x0, #16]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v4.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip1 v5.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT: stp q4, q5, [x0]
+; CHECK-GI-NEXT: stp q0, q1, [x0, #32]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <2 x ptr> %v0, <2 x ptr> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%s1 = shufflevector <2 x ptr> %v2, <2 x ptr> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%interleaved.vec = shufflevector <4 x ptr> %s0, <4 x ptr> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
@@ -605,6 +798,15 @@ define void @load_undef_mask_factor2(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_undef_mask_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <8 x i32>, ptr %ptr, align 4
%v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 poison, i32 2, i32 poison, i32 6>
%v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 7>
@@ -659,6 +861,24 @@ define void @load_undef_mask_factor3(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_undef_mask_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: dup v0.4s, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[3]
+; CHECK-GI-NEXT: ldr q1, [x0, #32]
+; CHECK-GI-NEXT: mov v2.s[3], v1.s[1]
+; CHECK-GI-NEXT: mov v3.s[3], v1.s[2]
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <12 x i32>, ptr %ptr, align 4
%v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -716,6 +936,21 @@ define void @load_undef_mask_factor4(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: // fake_use: $w0
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_undef_mask_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: adrp x8, .LCPI14_0
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT: zip1 v3.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: trn2 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v5.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q5
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
@@ -748,6 +983,12 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
; NO_NEON-NEXT: stp w3, w7, [x0, #16]
; NO_NEON-NEXT: stp w4, w8, [x0, #24]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q0, [x0, #16]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -798,6 +1039,28 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: orr x9, x10, x5, lsl #32
; NO_NEON-NEXT: stp x9, x11, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI16_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: mov s5, v2.s[2]
+; CHECK-GI-NEXT: mov v6.s[1], v2.s[1]
+; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI16_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v3.16b, { v0.16b, v1.16b }, v3.16b
+; CHECK-GI-NEXT: mov v5.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v6.s[2], v0.s[2]
+; CHECK-GI-NEXT: mov s4, v3.s[0]
+; CHECK-GI-NEXT: mov v5.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v6.s[3], v1.s[2]
+; CHECK-GI-NEXT: mov v4.s[1], v3.s[1]
+; CHECK-GI-NEXT: mov v5.s[3], v2.s[3]
+; CHECK-GI-NEXT: mov v4.s[2], v3.s[2]
+; CHECK-GI-NEXT: str q5, [x0, #32]
+; CHECK-GI-NEXT: mov v4.s[3], v3.s[3]
+; CHECK-GI-NEXT: stp q4, q6, [x0]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -852,6 +1115,26 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: stp w1, w5, [x0]
; NO_NEON-NEXT: str w8, [x0, #8]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s4, v0.s[2]
+; CHECK-GI-NEXT: mov s5, v0.s[3]
+; CHECK-GI-NEXT: mov s0, v0.s[0]
+; CHECK-GI-NEXT: mov v6.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v6.s[2], v2.s[1]
+; CHECK-GI-NEXT: mov v4.s[2], v2.s[2]
+; CHECK-GI-NEXT: mov v5.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v0.s[2], v2.s[0]
+; CHECK-GI-NEXT: mov v6.s[3], v3.s[1]
+; CHECK-GI-NEXT: mov v4.s[3], v3.s[2]
+; CHECK-GI-NEXT: mov v5.s[3], v3.s[3]
+; CHECK-GI-NEXT: stp q0, q6, [x0]
+; CHECK-GI-NEXT: stp q4, q5, [x0, #32]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -859,27 +1142,27 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
ret void
}
-define void @load_illegal_factor2(ptr %ptr) nounwind {
+define <3 x float> @load_illegal_factor2(ptr %ptr) nounwind {
; NEON-LABEL: load_illegal_factor2:
; NEON: // %bb.0:
; NEON-NEXT: ldr q0, [x0]
; NEON-NEXT: uzp1 v0.4s, v0.4s, v0.4s
-; NEON-NEXT: // fake_use: $q0
; NEON-NEXT: ret
;
; NO_NEON-LABEL: load_illegal_factor2:
; NO_NEON: // %bb.0:
; NO_NEON-NEXT: ldr s0, [x0]
; NO_NEON-NEXT: ldr s1, [x0, #8]
-; NO_NEON-NEXT: // fake_use: $s0
-; NO_NEON-NEXT: // fake_use: $s1
-; NO_NEON-NEXT: // fake_use: $s0
-; NO_NEON-NEXT: // fake_use: $s0
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_illegal_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <3 x float>, ptr %ptr, align 16
%v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
- tail call void (...) @llvm.fake.use(<3 x float> %v0)
- ret void
+ ret <3 x float> %v0
}
define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
@@ -896,6 +1179,16 @@ define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
; NO_NEON-NEXT: orr x8, x8, x9, lsl #32
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_illegal_factor2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: add x8, x0, #4
+; CHECK-GI-NEXT: add x9, x0, #8
+; CHECK-GI-NEXT: str s0, [x0]
+; CHECK-GI-NEXT: st1 { v0.s }[1], [x8]
+; CHECK-GI-NEXT: st1 { v0.s }[2], [x9]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <3 x float> %v0, <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
store <3 x float> %interleaved.vec, ptr %ptr, align 16
ret void
@@ -941,6 +1234,20 @@ define void @store_general_mask_factor4(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1
; NO_NEON-NEXT: stp w5, w10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v1.s[0]
+; CHECK-GI-NEXT: mov s1, v1.s[1]
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT: mov v1.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[0]
+; CHECK-GI-NEXT: mov v1.s[2], v3.s[1]
+; CHECK-GI-NEXT: mov v0.s[3], v2.s[0]
+; CHECK-GI-NEXT: mov v1.s[3], v2.s[1]
+; CHECK-GI-NEXT: stp q0, q1, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -989,6 +1296,19 @@ define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: str w10, [x0, #4]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefbeg:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v1.s[1]
+; CHECK-GI-NEXT: mov v1.s[1], v4.s[0]
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: mov v0.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v1.s[2], v3.s[0]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[1]
+; CHECK-GI-NEXT: mov v1.s[3], v2.s[0]
+; CHECK-GI-NEXT: mov v0.s[3], v2.s[1]
+; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1033,6 +1353,19 @@ define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: stp w5, w9, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefend:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v1.s[0]
+; CHECK-GI-NEXT: mov s1, v1.s[1]
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT: mov v1.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[0]
+; CHECK-GI-NEXT: mov v1.s[2], v3.s[1]
+; CHECK-GI-NEXT: mov v0.s[3], v2.s[0]
+; CHECK-GI-NEXT: stp q0, q1, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1075,6 +1408,18 @@ define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x
; NO_NEON-NEXT: str w5, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefmid:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v1.s[0]
+; CHECK-GI-NEXT: mov s1, v1.s[1]
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[0]
+; CHECK-GI-NEXT: mov v1.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[3], v2.s[0]
+; CHECK-GI-NEXT: mov v1.s[3], v2.s[1]
+; CHECK-GI-NEXT: stp q0, q1, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1109,6 +1454,13 @@ define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: str w9, [x0, #28]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefmulti:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v1.s[3], v2.s[0]
+; CHECK-GI-NEXT: dup v0.4s, v2.s[1]
+; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
store <8 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1169,6 +1521,25 @@ define void @store_general_mask_factor3(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [sp]
+; CHECK-GI-NEXT: mov s2, v1.s[0]
+; CHECK-GI-NEXT: mov s5, v4.s[2]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[0]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v2.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v5.s[2], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v2.s[3], v1.s[1]
+; CHECK-GI-NEXT: mov v5.s[3], v4.s[3]
+; CHECK-GI-NEXT: mov v3.s[3], v0.s[2]
+; CHECK-GI-NEXT: str q5, [x0, #32]
+; CHECK-GI-NEXT: stp q2, q3, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1224,6 +1595,23 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_undefmultimid:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v4.s[2]
+; CHECK-GI-NEXT: ldr q2, [sp]
+; CHECK-GI-NEXT: mov s3, v1.s[0]
+; CHECK-GI-NEXT: mov s5, v2.s[1]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v5.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v3.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v5.s[3], v2.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[3]
+; CHECK-GI-NEXT: stp q3, q5, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1270,6 +1658,23 @@ define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_undef_fail:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s0, v4.s[2]
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: mov s1, v1.s[0]
+; CHECK-GI-NEXT: mov s5, v3.s[1]
+; CHECK-GI-NEXT: mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT: mov v5.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v0.s[2], v3.s[3]
+; CHECK-GI-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v5.s[3], v3.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[3]
+; CHECK-GI-NEXT: stp q1, q5, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1320,6 +1725,24 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: stp x9, x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_undeflane:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI28_2
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5
+; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI28_2]
+; CHECK-GI-NEXT: adrp x8, .LCPI28_1
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI28_1]
+; CHECK-GI-NEXT: adrp x8, .LCPI28_0
+; CHECK-GI-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-GI-NEXT: mov v5.16b, v3.16b
+; CHECK-GI-NEXT: tbl v2.16b, { v4.16b, v5.16b }, v2.16b
+; CHECK-GI-NEXT: stp q0, q1, [x0]
+; CHECK-GI-NEXT: str q2, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1365,6 +1788,22 @@ define void @store_general_mask_factor3_negativestart(ptr %ptr, <32 x i32> %v0,
; NO_NEON-NEXT: stp x8, x10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_negativestart:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s1, v4.s[2]
+; CHECK-GI-NEXT: ldr q2, [sp]
+; CHECK-GI-NEXT: mov s3, v2.s[1]
+; CHECK-GI-NEXT: mov v1.s[1], v0.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v3.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v1.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v3.s[3], v2.s[2]
+; CHECK-GI-NEXT: mov v1.s[3], v4.s[3]
+; CHECK-GI-NEXT: stp q0, q3, [x0]
+; CHECK-GI-NEXT: str q1, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
store <12 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
@@ -1390,6 +1829,18 @@ define void @no_interleave(<4 x float> %a0) {
; NO_NEON-NEXT: stp s0, s3, [x8]
; NO_NEON-NEXT: str s3, [x8, #8]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: no_interleave:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT: adrp x8, .LCPI30_0
+; CHECK-GI-NEXT: mov v1.16b, v0.16b
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI30_0]
+; CHECK-GI-NEXT: adrp x8, :got:g
+; CHECK-GI-NEXT: ldr x8, [x8, :got_lo12:g]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: str q0, [x8]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 3, i32 7, i32 poison>
store <4 x float> %v0, ptr @g, align 16
ret void
@@ -1447,6 +1898,20 @@ define void @load_factor2_wide2(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $w10
; NO_NEON-NEXT: // fake_use: $w8
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor2_wide2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q1, q0, [x0]
+; CHECK-GI-NEXT: ldp q3, q2, [x0, #32]
+; CHECK-GI-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: uzp1 v5.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q5
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <16 x i32>, ptr %ptr, align 4
%v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -1552,6 +2017,27 @@ define void @load_factor2_wide3(ptr %ptr) {
; NO_NEON-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; NO_NEON-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor2_wide3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q1, q0, [x0]
+; CHECK-GI-NEXT: ldp q3, q2, [x0, #32]
+; CHECK-GI-NEXT: ldp q5, q4, [x0, #64]
+; CHECK-GI-NEXT: uzp1 v16.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: uzp1 v7.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: uzp1 v6.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: // fake_use: $q16
+; CHECK-GI-NEXT: // fake_use: $q7
+; CHECK-GI-NEXT: // fake_use: $q6
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
@@ -1658,6 +2144,40 @@ define void @load_factor3_wide(ptr %ptr) {
; NO_NEON-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; NO_NEON-NEXT: ldr x23, [sp], #48 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor3_wide:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q4, [x0]
+; CHECK-GI-NEXT: ldr q16, [x0, #32]
+; CHECK-GI-NEXT: ldp q1, q5, [x0, #48]
+; CHECK-GI-NEXT: ldr q17, [x0, #80]
+; CHECK-GI-NEXT: mov v3.16b, v0.16b
+; CHECK-GI-NEXT: rev64 v7.4s, v4.4s
+; CHECK-GI-NEXT: mov v2.16b, v1.16b
+; CHECK-GI-NEXT: rev64 v6.4s, v5.4s
+; CHECK-GI-NEXT: mov v3.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v2.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v7.s[0], v0.s[1]
+; CHECK-GI-NEXT: mov v6.s[0], v1.s[1]
+; CHECK-GI-NEXT: mov v3.s[2], v4.s[2]
+; CHECK-GI-NEXT: mov v4.s[0], v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[2], v5.s[2]
+; CHECK-GI-NEXT: mov v5.s[0], v1.s[2]
+; CHECK-GI-NEXT: mov v7.s[3], v16.s[2]
+; CHECK-GI-NEXT: mov v6.s[3], v17.s[2]
+; CHECK-GI-NEXT: mov v4.s[2], v16.s[0]
+; CHECK-GI-NEXT: mov v3.s[3], v16.s[1]
+; CHECK-GI-NEXT: mov v5.s[2], v17.s[0]
+; CHECK-GI-NEXT: mov v2.s[3], v17.s[1]
+; CHECK-GI-NEXT: mov v4.s[3], v16.s[3]
+; CHECK-GI-NEXT: mov v5.s[3], v17.s[3]
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: // fake_use: $q7
+; CHECK-GI-NEXT: // fake_use: $q6
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q5
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <24 x i32>, ptr %ptr, align 4
%v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
%v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
@@ -1805,6 +2325,48 @@ define void @load_factor4_wide(ptr %ptr) {
; NO_NEON-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload
; NO_NEON-NEXT: add sp, sp, #112
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor4_wide:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q1, q0, [x0, #32]
+; CHECK-GI-NEXT: ldp q3, q2, [x0, #96]
+; CHECK-GI-NEXT: ldp q5, q4, [x0, #64]
+; CHECK-GI-NEXT: zip1 v17.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: ldp q7, q6, [x0]
+; CHECK-GI-NEXT: zip1 v16.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v23.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v24.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: uzp2 v20.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v18.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: trn2 v22.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v19.4s, v7.4s, v6.4s
+; CHECK-GI-NEXT: uzp2 v21.4s, v7.4s, v6.4s
+; CHECK-GI-NEXT: zip2 v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: ext v25.16b, v1.16b, v17.16b, #8
+; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: trn2 v1.4s, v7.4s, v6.4s
+; CHECK-GI-NEXT: ext v3.16b, v3.16b, v16.16b, #8
+; CHECK-GI-NEXT: zip2 v4.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: zip2 v6.4s, v7.4s, v6.4s
+; CHECK-GI-NEXT: uzp2 v5.4s, v20.4s, v5.4s
+; CHECK-GI-NEXT: mov v22.d[1], v16.d[1]
+; CHECK-GI-NEXT: uzp2 v7.4s, v21.4s, v7.4s
+; CHECK-GI-NEXT: mov v19.d[1], v25.d[1]
+; CHECK-GI-NEXT: mov v1.d[1], v17.d[1]
+; CHECK-GI-NEXT: mov v18.d[1], v3.d[1]
+; CHECK-GI-NEXT: mov v4.d[1], v23.d[1]
+; CHECK-GI-NEXT: mov v6.d[1], v24.d[1]
+; CHECK-GI-NEXT: mov v5.d[1], v2.d[1]
+; CHECK-GI-NEXT: mov v7.d[1], v0.d[1]
+; CHECK-GI-NEXT: // fake_use: $q19
+; CHECK-GI-NEXT: // fake_use: $q18
+; CHECK-GI-NEXT: // fake_use: $q1
+; CHECK-GI-NEXT: // fake_use: $q22
+; CHECK-GI-NEXT: // fake_use: $q6
+; CHECK-GI-NEXT: // fake_use: $q4
+; CHECK-GI-NEXT: // fake_use: $q7
+; CHECK-GI-NEXT: // fake_use: $q5
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <32 x i32>, ptr %ptr, align 4
%v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%v1 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
@@ -1817,8 +2379,8 @@ define void @load_factor4_wide(ptr %ptr) {
ret void
}
-define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
-; NEON-IAENABLED-LABEL: store_factor2_wide:
+define void @store_factor2_wide2(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
+; NEON-IAENABLED-LABEL: store_factor2_wide2:
; NEON-IAENABLED: // %bb.0:
; NEON-IAENABLED-NEXT: mov v5.16b, v2.16b
; NEON-IAENABLED-NEXT: // kill: def $q3 killed $q3 def $q2_q3
@@ -1828,7 +2390,7 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
; NEON-IAENABLED-NEXT: st2 { v2.4s, v3.4s }, [x0]
; NEON-IAENABLED-NEXT: ret
;
-; NEON-IADISABLED-LABEL: store_factor2_wide:
+; NEON-IADISABLED-LABEL: store_factor2_wide2:
; NEON-IADISABLED: // %bb.0:
; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
@@ -1838,7 +2400,7 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
; NEON-IADISABLED-NEXT: stp q0, q3, [x0]
; NEON-IADISABLED-NEXT: ret
;
-; NO_NEON-LABEL: store_factor2_wide:
+; NO_NEON-LABEL: store_factor2_wide2:
; NO_NEON: // %bb.0:
; NO_NEON-NEXT: ldr w8, [sp, #64]
; NO_NEON-NEXT: ldr w9, [sp]
@@ -1858,11 +2420,155 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
; NO_NEON-NEXT: stp w2, w8, [x0, #8]
; NO_NEON-NEXT: stp w1, w9, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_wide2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v4.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: stp q4, q0, [x0]
+; CHECK-GI-NEXT: stp q2, q1, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
store <16 x i32> %interleaved.vec, ptr %ptr, align 4
ret void
}
+define void @store_factor2_wide15(ptr %ptr, ptr %p0, ptr %p1) {
+; NEON-LABEL: store_factor2_wide15:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q1, q0, [x1]
+; NEON-NEXT: ldp q4, q3, [x2]
+; NEON-NEXT: ext v2.16b, v1.16b, v0.16b, #12
+; NEON-NEXT: ext v5.16b, v4.16b, v3.16b, #12
+; NEON-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; NEON-NEXT: zip2 v3.4s, v1.4s, v4.4s
+; NEON-NEXT: ext v2.16b, v1.16b, v2.16b, #12
+; NEON-NEXT: zip1 v1.4s, v1.4s, v4.4s
+; NEON-NEXT: ext v5.16b, v4.16b, v5.16b, #8
+; NEON-NEXT: rev64 v2.4s, v2.4s
+; NEON-NEXT: str q1, [x0]
+; NEON-NEXT: trn2 v2.4s, v2.4s, v5.4s
+; NEON-NEXT: ext v0.16b, v2.16b, v0.16b, #8
+; NEON-NEXT: mov v3.d[1], v2.d[0]
+; NEON-NEXT: stp q3, q0, [x0, #16]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor2_wide15:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x8, x11, [x2, #8]
+; NO_NEON-NEXT: ldr x14, [x1]
+; NO_NEON-NEXT: ldp x9, x10, [x1, #8]
+; NO_NEON-NEXT: ldr x12, [x2]
+; NO_NEON-NEXT: mov x13, x8
+; NO_NEON-NEXT: bfxil x13, x9, #32, #32
+; NO_NEON-NEXT: bfi x9, x8, #32, #32
+; NO_NEON-NEXT: mov x8, x11
+; NO_NEON-NEXT: bfxil x8, x10, #32, #32
+; NO_NEON-NEXT: bfi x10, x11, #32, #32
+; NO_NEON-NEXT: stp x9, x13, [x0, #16]
+; NO_NEON-NEXT: mov x9, x14
+; NO_NEON-NEXT: bfi x9, x12, #32, #32
+; NO_NEON-NEXT: bfxil x12, x14, #32, #32
+; NO_NEON-NEXT: stp x10, x8, [x0, #32]
+; NO_NEON-NEXT: stp x9, x12, [x0]
+; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_wide15:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [x1]
+; CHECK-GI-NEXT: ldr q1, [x2]
+; CHECK-GI-NEXT: ldr d2, [x1, #16]
+; CHECK-GI-NEXT: ldr d3, [x2, #16]
+; CHECK-GI-NEXT: zip1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip1 v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: stp q4, q0, [x0]
+; CHECK-GI-NEXT: str q1, [x0, #32]
+; CHECK-GI-NEXT: ret
+ %v0 = load <6 x i32>, ptr %p0
+ %v1 = load <6 x i32>, ptr %p1
+ %interleaved.vec = shufflevector <6 x i32> %v0, <6 x i32> %v1, <12 x i32> <i32 0, i32 6, i32 1, i32 7, i32 2, i32 8, i32 3, i32 9, i32 4, i32 10, i32 5, i32 11>
+ store <12 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor2_wide3(ptr %ptr, ptr %p0, ptr %p1) {
+; NEON-LABEL: store_factor2_wide3:
+; NEON: // %bb.0:
+; NEON-NEXT: ldp q0, q2, [x1, #16]
+; NEON-NEXT: ldr q3, [x1]
+; NEON-NEXT: ldp q1, q4, [x2, #16]
+; NEON-NEXT: ldr q5, [x2]
+; NEON-NEXT: zip1 v6.4s, v0.4s, v1.4s
+; NEON-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; NEON-NEXT: zip1 v1.4s, v2.4s, v4.4s
+; NEON-NEXT: zip2 v2.4s, v2.4s, v4.4s
+; NEON-NEXT: zip2 v4.4s, v3.4s, v5.4s
+; NEON-NEXT: zip1 v3.4s, v3.4s, v5.4s
+; NEON-NEXT: stp q6, q0, [x0, #32]
+; NEON-NEXT: stp q3, q4, [x0]
+; NEON-NEXT: stp q1, q2, [x0, #64]
+; NEON-NEXT: ret
+;
+; NO_NEON-LABEL: store_factor2_wide3:
+; NO_NEON: // %bb.0:
+; NO_NEON-NEXT: ldp x15, x10, [x2]
+; NO_NEON-NEXT: ldp x14, x13, [x1]
+; NO_NEON-NEXT: ldp x17, x18, [x2, #16]
+; NO_NEON-NEXT: ldp x8, x9, [x1, #16]
+; NO_NEON-NEXT: mov x16, x10
+; NO_NEON-NEXT: bfxil x16, x13, #32, #32
+; NO_NEON-NEXT: bfi x13, x10, #32, #32
+; NO_NEON-NEXT: ldp x11, x12, [x1, #32]
+; NO_NEON-NEXT: ldp x10, x1, [x2, #32]
+; NO_NEON-NEXT: mov x2, x17
+; NO_NEON-NEXT: bfxil x2, x8, #32, #32
+; NO_NEON-NEXT: bfi x8, x17, #32, #32
+; NO_NEON-NEXT: stp x13, x16, [x0, #16]
+; NO_NEON-NEXT: mov x13, x18
+; NO_NEON-NEXT: bfxil x13, x9, #32, #32
+; NO_NEON-NEXT: bfi x9, x18, #32, #32
+; NO_NEON-NEXT: stp x8, x2, [x0, #32]
+; NO_NEON-NEXT: mov x8, x10
+; NO_NEON-NEXT: bfxil x8, x11, #32, #32
+; NO_NEON-NEXT: bfi x11, x10, #32, #32
+; NO_NEON-NEXT: stp x9, x13, [x0, #48]
+; NO_NEON-NEXT: mov x9, x1
+; NO_NEON-NEXT: mov x10, x14
+; NO_NEON-NEXT: bfxil x9, x12, #32, #32
+; NO_NEON-NEXT: bfi x12, x1, #32, #32
+; NO_NEON-NEXT: bfi x10, x15, #32, #32
+; NO_NEON-NEXT: bfxil x15, x14, #32, #32
+; NO_NEON-NEXT: stp x11, x8, [x0, #64]
+; NO_NEON-NEXT: stp x12, x9, [x0, #80]
+; NO_NEON-NEXT: stp x10, x15, [x0]
+; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_wide3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q3, [x2]
+; CHECK-GI-NEXT: ldr q5, [x2, #32]
+; CHECK-GI-NEXT: ldp q1, q2, [x1]
+; CHECK-GI-NEXT: zip1 v4.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: ldr q1, [x1, #32]
+; CHECK-GI-NEXT: zip1 v6.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: zip2 v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: zip1 v3.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT: stp q4, q0, [x0]
+; CHECK-GI-NEXT: stp q6, q2, [x0, #32]
+; CHECK-GI-NEXT: stp q3, q1, [x0, #64]
+; CHECK-GI-NEXT: ret
+ %v0 = load <12 x i32>, ptr %p0
+ %v1 = load <12 x i32>, ptr %p1
+ %interleaved.vec = shufflevector <12 x i32> %v0, <12 x i32> %v1, <24 x i32> <i32 0, i32 12, i32 1, i32 13, i32 2, i32 14, i32 3, i32 15, i32 4, i32 16, i32 5, i32 17, i32 6, i32 18, i32 7, i32 19, i32 8, i32 20, i32 9, i32 21, i32 10, i32 22, i32 11, i32 23>
+ store <24 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
; NEON-IAENABLED-LABEL: store_factor3_wide:
; NEON-IAENABLED: // %bb.0:
@@ -1951,6 +2657,37 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
; NO_NEON-NEXT: str x9, [x0, #24]
; NO_NEON-NEXT: stp x8, x12, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3_wide:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s6, v0.s[0]
+; CHECK-GI-NEXT: mov s7, v2.s[1]
+; CHECK-GI-NEXT: mov s16, v4.s[2]
+; CHECK-GI-NEXT: mov s17, v1.s[0]
+; CHECK-GI-NEXT: mov s18, v3.s[1]
+; CHECK-GI-NEXT: mov s19, v5.s[2]
+; CHECK-GI-NEXT: mov v6.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v7.s[1], v4.s[1]
+; CHECK-GI-NEXT: mov v16.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v17.s[1], v3.s[0]
+; CHECK-GI-NEXT: mov v18.s[1], v5.s[1]
+; CHECK-GI-NEXT: mov v19.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v6.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v7.s[2], v0.s[2]
+; CHECK-GI-NEXT: mov v16.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v17.s[2], v5.s[0]
+; CHECK-GI-NEXT: mov v18.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v19.s[2], v3.s[3]
+; CHECK-GI-NEXT: mov v6.s[3], v0.s[1]
+; CHECK-GI-NEXT: mov v7.s[3], v2.s[2]
+; CHECK-GI-NEXT: mov v16.s[3], v4.s[3]
+; CHECK-GI-NEXT: mov v17.s[3], v1.s[1]
+; CHECK-GI-NEXT: mov v18.s[3], v3.s[2]
+; CHECK-GI-NEXT: mov v19.s[3], v5.s[3]
+; CHECK-GI-NEXT: stp q6, q7, [x0]
+; CHECK-GI-NEXT: stp q16, q17, [x0, #32]
+; CHECK-GI-NEXT: stp q18, q19, [x0, #64]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
@@ -2061,6 +2798,46 @@ define void @store_factor4_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
; NO_NEON-NEXT: ldr w8, [sp, #8]
; NO_NEON-NEXT: stp w8, w9, [x0, #4]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4_wide:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov s16, v0.s[0]
+; CHECK-GI-NEXT: mov s17, v0.s[1]
+; CHECK-GI-NEXT: mov s18, v0.s[2]
+; CHECK-GI-NEXT: mov s0, v0.s[3]
+; CHECK-GI-NEXT: mov s19, v1.s[0]
+; CHECK-GI-NEXT: mov s20, v1.s[1]
+; CHECK-GI-NEXT: mov s21, v1.s[2]
+; CHECK-GI-NEXT: mov s1, v1.s[3]
+; CHECK-GI-NEXT: mov v16.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v17.s[1], v2.s[1]
+; CHECK-GI-NEXT: mov v18.s[1], v2.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v2.s[3]
+; CHECK-GI-NEXT: mov v19.s[1], v3.s[0]
+; CHECK-GI-NEXT: mov v20.s[1], v3.s[1]
+; CHECK-GI-NEXT: mov v21.s[1], v3.s[2]
+; CHECK-GI-NEXT: mov v1.s[1], v3.s[3]
+; CHECK-GI-NEXT: mov v16.s[2], v4.s[0]
+; CHECK-GI-NEXT: mov v17.s[2], v4.s[1]
+; CHECK-GI-NEXT: mov v18.s[2], v4.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v4.s[3]
+; CHECK-GI-NEXT: mov v19.s[2], v5.s[0]
+; CHECK-GI-NEXT: mov v20.s[2], v5.s[1]
+; CHECK-GI-NEXT: mov v21.s[2], v5.s[2]
+; CHECK-GI-NEXT: mov v1.s[2], v5.s[3]
+; CHECK-GI-NEXT: mov v16.s[3], v6.s[0]
+; CHECK-GI-NEXT: mov v17.s[3], v6.s[1]
+; CHECK-GI-NEXT: mov v18.s[3], v6.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v6.s[3]
+; CHECK-GI-NEXT: mov v19.s[3], v7.s[0]
+; CHECK-GI-NEXT: mov v20.s[3], v7.s[1]
+; CHECK-GI-NEXT: mov v21.s[3], v7.s[2]
+; CHECK-GI-NEXT: mov v1.s[3], v7.s[3]
+; CHECK-GI-NEXT: stp q16, q17, [x0]
+; CHECK-GI-NEXT: stp q18, q0, [x0, #32]
+; CHECK-GI-NEXT: stp q19, q20, [x0, #64]
+; CHECK-GI-NEXT: stp q21, q1, [x0, #96]
+; CHECK-GI-NEXT: ret
%s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
@@ -2088,6 +2865,16 @@ define void @load_factor2_fp128(ptr %ptr) {
; NO_NEON-NEXT: // fake_use: $q1
; NO_NEON-NEXT: // fake_use: $q2
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_factor2_fp128:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q3, q2, [x0, #32]
+; CHECK-GI-NEXT: // fake_use: $q0
+; CHECK-GI-NEXT: // fake_use: $q3
+; CHECK-GI-NEXT: // fake_use: $q1
+; CHECK-GI-NEXT: // fake_use: $q2
+; CHECK-GI-NEXT: ret
%interleaved.vec = load <4 x fp128>, ptr %ptr, align 16
%v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 0, i32 2>
%v1 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> poison, <2 x i32> <i32 1, i32 3>
@@ -2141,6 +2928,24 @@ define <4 x i1> @load_large_vector(ptr %p) {
; NO_NEON-NEXT: cmp x9, x8
; NO_NEON-NEXT: cset w3, ne
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_large_vector:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q1, q2, [x0, #32]
+; CHECK-GI-NEXT: add x8, x0, #24
+; CHECK-GI-NEXT: ldr q0, [x0, #80]
+; CHECK-GI-NEXT: ldr q3, [x0]
+; CHECK-GI-NEXT: add x9, x0, #72
+; CHECK-GI-NEXT: ext v1.16b, v3.16b, v1.16b, #8
+; CHECK-GI-NEXT: ext v0.16b, v2.16b, v0.16b, #8
+; CHECK-GI-NEXT: ld1 { v3.d }[1], [x8]
+; CHECK-GI-NEXT: ld1 { v2.d }[1], [x9]
+; CHECK-GI-NEXT: cmeq v0.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT: cmeq v1.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: mvn v0.16b, v0.16b
+; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: ret
%l = load <12 x ptr>, ptr %p
%s1 = shufflevector <12 x ptr> %l, <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
%s2 = shufflevector <12 x ptr> %l, <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -2175,6 +2980,17 @@ define <4 x i1> @load_large_vector_intrinsic(ptr %p) {
; NO_NEON-NEXT: cmp x9, x8
; NO_NEON-NEXT: cset w3, ne
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: load_large_vector_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ld3 { v0.2d, v1.2d, v2.2d }, [x0], #48
+; CHECK-GI-NEXT: ld3 { v3.2d, v4.2d, v5.2d }, [x0]
+; CHECK-GI-NEXT: cmeq v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: cmeq v1.2d, v3.2d, v4.2d
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mvn v0.16b, v0.16b
+; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: ret
%intrinsic.load.0 = load <6 x i64>, ptr %p, align 1
%ldN = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.vector.deinterleave3.v6i64(<6 x i64> %intrinsic.load.0)
%1 = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } %ldN, 1
@@ -2206,9 +3022,9 @@ define void @store_factor2_intrinsic(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
; NEON-IADISABLED: // %bb.0:
; NEON-IADISABLED-NEXT: // kill: def $d0 killed $d0 def $q0
; NEON-IADISABLED-NEXT: // kill: def $d1 killed $d1 def $q1
-; NEON-IADISABLED-NEXT: adrp x8, .LCPI41_0
+; NEON-IADISABLED-NEXT: adrp x8, .LCPI43_0
; NEON-IADISABLED-NEXT: mov v0.d[1], v1.d[0]
-; NEON-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI41_0]
+; NEON-IADISABLED-NEXT: ldr q1, [x8, :lo12:.LCPI43_0]
; NEON-IADISABLED-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; NEON-IADISABLED-NEXT: str q0, [x0]
; NEON-IADISABLED-NEXT: ret
@@ -2241,6 +3057,14 @@ define void @store_factor2_intrinsic(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
; NO_NEON-NEXT: strb w9, [x0, #1]
; NO_NEON-NEXT: strb w1, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT: zip1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: str q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i8> @llvm.vector.interleave2.v16i8(<8 x i8> %v0, <8 x i8> %v1)
store <16 x i8> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2287,6 +3111,16 @@ define void @store_factor2_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1
; NO_NEON-NEXT: stp w2, w8, [x0, #8]
; NO_NEON-NEXT: stp w1, w9, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_wide_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v4.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: stp q4, q0, [x0]
+; CHECK-GI-NEXT: stp q2, q1, [x0, #32]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2325,6 +3159,14 @@ define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: stp x8, x12, [x0, #16]
; NO_NEON-NEXT: stp x10, x9, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
store <12 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2391,6 +3233,18 @@ define void @store_factor3_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1
; NO_NEON-NEXT: str x9, [x0, #24]
; NO_NEON-NEXT: stp x8, x12, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3_wide_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v18.16b, v4.16b
+; CHECK-GI-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; CHECK-GI-NEXT: mov v17.16b, v2.16b
+; CHECK-GI-NEXT: mov v4.16b, v3.16b
+; CHECK-GI-NEXT: mov v16.16b, v0.16b
+; CHECK-GI-NEXT: mov v3.16b, v1.16b
+; CHECK-GI-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-GI-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
store <24 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2429,6 +3283,15 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
; NO_NEON-NEXT: stp w5, w9, [x0, #4]
; NO_NEON-NEXT: str w1, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2502,6 +3365,20 @@ define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1
; NO_NEON-NEXT: ldr w8, [sp, #8]
; NO_NEON-NEXT: stp w8, w9, [x0, #4]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4_wide_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; CHECK-GI-NEXT: mov v19.16b, v6.16b
+; CHECK-GI-NEXT: mov v18.16b, v4.16b
+; CHECK-GI-NEXT: mov v6.16b, v5.16b
+; CHECK-GI-NEXT: mov v17.16b, v2.16b
+; CHECK-GI-NEXT: mov v5.16b, v3.16b
+; CHECK-GI-NEXT: mov v16.16b, v0.16b
+; CHECK-GI-NEXT: mov v4.16b, v1.16b
+; CHECK-GI-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; CHECK-GI-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <32 x i32> @llvm.vector.interleave4.v32i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3)
store <32 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2545,6 +3422,14 @@ define void @store_general_mask_factor3_intrinsic(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; CHECK-GI-NEXT: mov v2.16b, v1.16b
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
@@ -2591,6 +3476,14 @@ define void @store_general_mask_factor3_undeflane_intrinsic(ptr %ptr, <32 x i32>
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_undeflane_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; CHECK-GI-NEXT: mov v2.16b, v0.16b
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
@@ -2637,6 +3530,14 @@ define void @store_general_mask_factor3_undefmultimid_intrinsic(ptr %ptr, <32 x
; NO_NEON-NEXT: str x8, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor3_undefmultimid_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q2_q3_q4
+; CHECK-GI-NEXT: mov v2.16b, v1.16b
+; CHECK-GI-NEXT: ldr q3, [sp]
+; CHECK-GI-NEXT: st3 { v2.4s, v3.4s, v4.4s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
@@ -2673,6 +3574,15 @@ define void @store_general_mask_factor4_intrinsic(ptr %ptr, <32 x i32> %v0, <32
; NO_NEON-NEXT: stp w5, w10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; CHECK-GI-NEXT: mov v6.16b, v2.16b
+; CHECK-GI-NEXT: mov v3.16b, v1.16b
+; CHECK-GI-NEXT: ldr d5, [sp]
+; CHECK-GI-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
@@ -2710,6 +3620,15 @@ define void @store_general_mask_factor4_undefbeg_intrinsic(ptr %ptr, <32 x i32>
; NO_NEON-NEXT: stp w5, w10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefbeg_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; CHECK-GI-NEXT: mov v6.16b, v2.16b
+; CHECK-GI-NEXT: mov v3.16b, v1.16b
+; CHECK-GI-NEXT: ldr d5, [sp]
+; CHECK-GI-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
@@ -2747,6 +3666,15 @@ define void @store_general_mask_factor4_undefend_intrinsic(ptr %ptr, <32 x i32>
; NO_NEON-NEXT: stp w5, w10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefend_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; CHECK-GI-NEXT: mov v6.16b, v2.16b
+; CHECK-GI-NEXT: mov v3.16b, v1.16b
+; CHECK-GI-NEXT: ldr d5, [sp]
+; CHECK-GI-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
@@ -2784,6 +3712,15 @@ define void @store_general_mask_factor4_undefmid_intrinsic(ptr %ptr, <32 x i32>
; NO_NEON-NEXT: stp w5, w10, [x0]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefmid_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q4 killed $q4 def $q3_q4_q5_q6
+; CHECK-GI-NEXT: mov v6.16b, v2.16b
+; CHECK-GI-NEXT: mov v3.16b, v1.16b
+; CHECK-GI-NEXT: ldr d5, [sp]
+; CHECK-GI-NEXT: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
@@ -2816,6 +3753,15 @@ define void @store_general_mask_factor4_undefmulti_intrinsic(ptr %ptr, <32 x i32
; NO_NEON-NEXT: stp w1, w8, [x0, #8]
; NO_NEON-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_general_mask_factor4_undefmulti_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v5.16b, v2.16b
+; CHECK-GI-NEXT: mov v2.16b, v1.16b
+; CHECK-GI-NEXT: mov v3.16b, v0.16b
+; CHECK-GI-NEXT: fmov d4, d3
+; CHECK-GI-NEXT: st4 { v2.2s, v3.2s, v4.2s, v5.2s }, [x0]
+; CHECK-GI-NEXT: ret
%1 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
%2 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
%3 = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
@@ -2845,6 +3791,13 @@ define void @store_ptrvec_factor2_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %
; NO_NEON-NEXT: stp x3, x5, [x0, #16]
; NO_NEON-NEXT: stp x2, x4, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor2_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <4 x ptr> @llvm.vector.interleave2.v4p0(<2 x ptr> %v0, <2 x ptr> %v1)
store <4 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2865,6 +3818,14 @@ define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %
; NO_NEON-NEXT: stp x6, x3, [x0, #16]
; NO_NEON-NEXT: stp x2, x4, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor3_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.2d, v1.2d, v2.2d }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <6 x ptr> @llvm.vector.interleave3.v6p0(<2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2)
store <6 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2889,6 +3850,15 @@ define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %
; NO_NEON-NEXT: ldr x8, [sp]
; NO_NEON-NEXT: stp x6, x8, [x0, #16]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_ptrvec_factor4_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <8 x ptr> @llvm.vector.interleave4.v8p0(<2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2, <2 x ptr> %v3)
store <8 x ptr> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2917,6 +3887,13 @@ define void @store_undef_mask_factor2_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i3
; NO_NEON-NEXT: stp w4, w8, [x0, #24]
; NO_NEON-NEXT: stp w1, w5, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor2_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
store <8 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2955,6 +3932,14 @@ define void @store_undef_mask_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i3
; NO_NEON-NEXT: stp x8, x12, [x0, #16]
; NO_NEON-NEXT: stp x10, x9, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor3_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-GI-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
store <12 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
@@ -2993,6 +3978,15 @@ define void @store_undef_mask_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i3
; NO_NEON-NEXT: stp w5, w9, [x0, #4]
; NO_NEON-NEXT: str w1, [x0]
; NO_NEON-NEXT: ret
+;
+; CHECK-GI-LABEL: store_undef_mask_factor4_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0]
+; CHECK-GI-NEXT: ret
%interleaved.intrinsic = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3)
store <16 x i32> %interleaved.intrinsic, ptr %ptr, align 4
ret void
diff --git a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
index 963c28aeba32b..5926cb61572f5 100644
--- a/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/vldn_shuffle.ll
@@ -1,6 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64 --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64 --lower-interleaved-accesses=false -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for vld3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vld4_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vld3_multiuse_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vld4_multiuse_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor3_intrinsic
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for store_factor4_intrinsic
define void @vld2(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32 %numSamples) {
; CHECK-IAENABLED-LABEL: vld2:
@@ -37,6 +45,24 @@ define void @vld2(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-IADISABLED-NEXT: b.ne .LBB0_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2:
+; CHECK-GI: .Lfunc_begin0:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB0_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0], #32
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q0, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB0_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -110,6 +136,40 @@ define void @vld3(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-IADISABLED-NEXT: b.ne .LBB1_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3:
+; CHECK-GI: .Lfunc_begin1:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB1_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldr q4, [x0, #32]
+; CHECK-GI-NEXT: add x0, x0, #48
+; CHECK-GI-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov s0, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-GI-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[3]
+; CHECK-GI-NEXT: fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: str q0, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB1_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -195,6 +255,48 @@ define void @vld4(ptr nocapture readonly %pSrc, ptr noalias nocapture %pDst, i32
; CHECK-IADISABLED-NEXT: b.ne .LBB2_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4:
+; CHECK-GI: .Lfunc_begin2:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB2_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: add x9, x1, x8
+; CHECK-GI-NEXT: ldp q5, q6, [x0, #32]
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: add x0, x0, #64
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmul v5.4s, v5.4s, v5.4s
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov s4, v0.s[2]
+; CHECK-GI-NEXT: mov s0, v0.s[3]
+; CHECK-GI-NEXT: mov v2.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
+; CHECK-GI-NEXT: fmul v1.4s, v6.4s, v6.4s
+; CHECK-GI-NEXT: mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT: mov v3.s[2], v5.s[1]
+; CHECK-GI-NEXT: mov v4.s[2], v5.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v5.s[3]
+; CHECK-GI-NEXT: mov v2.s[3], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[3], v1.s[1]
+; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v1.s[3]
+; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: fadd v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: stp q2, q0, [x9]
+; CHECK-GI-NEXT: b.ne .LBB2_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -266,6 +368,27 @@ define void @twosrc(ptr nocapture readonly %pSrc, ptr nocapture readonly %pSrc2,
; CHECK-IADISABLED-NEXT: b.ne .LBB3_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc:
+; CHECK-GI: .Lfunc_begin3:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB3_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: add x9, x0, x8
+; CHECK-GI-NEXT: add x10, x1, x8
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: ldp q0, q1, [x9]
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: ldp q2, q3, [x10]
+; CHECK-GI-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q0, [x2], #16
+; CHECK-GI-NEXT: b.ne .LBB3_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -327,6 +450,24 @@ define void @vld2_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %
; CHECK-IADISABLED-NEXT: b.ne .LBB4_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2_multiuse:
+; CHECK-GI: .Lfunc_begin4:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB4_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0], #32
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: faddp v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q0, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB4_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -399,6 +540,40 @@ define void @vld3_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %
; CHECK-IADISABLED-NEXT: b.ne .LBB5_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3_multiuse:
+; CHECK-GI: .Lfunc_begin5:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB5_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldr q4, [x0, #32]
+; CHECK-GI-NEXT: add x0, x0, #48
+; CHECK-GI-NEXT: fmul v4.4s, v4.4s, v4.4s
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[3]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov s0, v0.s[2]
+; CHECK-GI-NEXT: mov v2.s[2], v1.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v1.s[3]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v2.s[3], v4.s[1]
+; CHECK-GI-NEXT: mov v3.s[3], v4.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mov v0.s[3], v4.s[3]
+; CHECK-GI-NEXT: fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: str q0, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB5_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -482,6 +657,48 @@ define void @vld4_multiuse(ptr nocapture readonly %pSrc, ptr noalias nocapture %
; CHECK-IADISABLED-NEXT: b.ne .LBB6_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4_multiuse:
+; CHECK-GI: .Lfunc_begin6:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB6_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: add x9, x1, x8
+; CHECK-GI-NEXT: ldp q5, q6, [x0, #32]
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: add x0, x0, #64
+; CHECK-GI-NEXT: fmul v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmul v5.4s, v5.4s, v5.4s
+; CHECK-GI-NEXT: mov s2, v0.s[0]
+; CHECK-GI-NEXT: mov s3, v0.s[1]
+; CHECK-GI-NEXT: mov s4, v0.s[2]
+; CHECK-GI-NEXT: mov s0, v0.s[3]
+; CHECK-GI-NEXT: mov v2.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[3]
+; CHECK-GI-NEXT: fmul v1.4s, v6.4s, v6.4s
+; CHECK-GI-NEXT: mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT: mov v3.s[2], v5.s[1]
+; CHECK-GI-NEXT: mov v4.s[2], v5.s[2]
+; CHECK-GI-NEXT: mov v0.s[2], v5.s[3]
+; CHECK-GI-NEXT: mov v2.s[3], v1.s[0]
+; CHECK-GI-NEXT: mov v3.s[3], v1.s[1]
+; CHECK-GI-NEXT: mov v4.s[3], v1.s[2]
+; CHECK-GI-NEXT: mov v0.s[3], v1.s[3]
+; CHECK-GI-NEXT: fadd v1.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: fadd v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: stp q2, q0, [x9]
+; CHECK-GI-NEXT: b.ne .LBB6_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -550,6 +767,26 @@ define void @transpose_s16_8x8_simpler(ptr nocapture noundef %a) {
; CHECK-IADISABLED-NEXT: str q2, [x0]
; CHECK-IADISABLED-NEXT: str q0, [x0, #64]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_simpler:
+; CHECK-GI: .Lfunc_begin7:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-GI-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-GI-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-GI-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q2, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #64]
+; CHECK-GI-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -623,6 +860,30 @@ define void @transpose_s16_8x8_simpler2(ptr nocapture noundef %a) {
; CHECK-IADISABLED-NEXT: str q2, [x0]
; CHECK-IADISABLED-NEXT: str q0, [x0, #64]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_simpler2:
+; CHECK-GI: .Lfunc_begin8:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: adrp x8, .LCPI8_1
+; CHECK-GI-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-GI-NEXT: adrp x9, .LCPI8_0
+; CHECK-GI-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-GI-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-GI-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn1 v2.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v1.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI8_1]
+; CHECK-GI-NEXT: ldr q5, [x9, :lo12:.LCPI8_0]
+; CHECK-GI-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-GI-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v5.16b
+; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q2, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #64]
+; CHECK-GI-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -745,6 +1006,52 @@ define void @transpose_s16_8x8(ptr nocapture noundef %0, ptr nocapture noundef %
; CHECK-IADISABLED-NEXT: str q2, [x6]
; CHECK-IADISABLED-NEXT: str q0, [x7]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8:
+; CHECK-GI: .Lfunc_begin9:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: ldr q1, [x1]
+; CHECK-GI-NEXT: ldr q2, [x2]
+; CHECK-GI-NEXT: ldr q3, [x3]
+; CHECK-GI-NEXT: ldr q4, [x4]
+; CHECK-GI-NEXT: ldr q5, [x5]
+; CHECK-GI-NEXT: trn1 v16.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn2 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: ldr q6, [x6]
+; CHECK-GI-NEXT: ldr q7, [x7]
+; CHECK-GI-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v17.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn2 v2.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn2 v3.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn1 v18.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn2 v4.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn1 v5.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn1 v7.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn1 v6.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn1 v19.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: trn2 v1.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn2 v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip1 v4.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: zip1 v17.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip1 v18.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip2 v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: str q3, [x0]
+; CHECK-GI-NEXT: zip2 v3.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: str q4, [x1]
+; CHECK-GI-NEXT: str q17, [x2]
+; CHECK-GI-NEXT: str q18, [x3]
+; CHECK-GI-NEXT: str q5, [x4]
+; CHECK-GI-NEXT: str q3, [x5]
+; CHECK-GI-NEXT: str q1, [x6]
+; CHECK-GI-NEXT: str q0, [x7]
+; CHECK-GI-NEXT: ret
%9 = load <8 x i16>, ptr %0, align 16
%10 = load <8 x i16>, ptr %1, align 16
%11 = shufflevector <8 x i16> %9, <8 x i16> %10, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
@@ -834,6 +1141,44 @@ define void @transpose_s16_8x8_(ptr nocapture noundef %0) {
; CHECK-NEXT: stp q1, q0, [x0, #96]
; CHECK-NEXT: stp q5, q3, [x0, #64]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_:
+; CHECK-GI: .Lfunc_begin10:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-GI-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-GI-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-GI-NEXT: trn1 v16.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn2 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn2 v2.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v17.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn2 v3.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn1 v18.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn2 v4.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn1 v5.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn1 v7.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn1 v6.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn1 v19.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: trn2 v1.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn2 v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip1 v4.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: zip1 v17.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip1 v18.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip2 v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: stp q3, q4, [x0]
+; CHECK-GI-NEXT: zip2 v3.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: stp q17, q18, [x0, #32]
+; CHECK-GI-NEXT: stp q1, q0, [x0, #96]
+; CHECK-GI-NEXT: stp q5, q3, [x0, #64]
+; CHECK-GI-NEXT: ret
%2 = load <8 x i16>, ptr %0, align 16
%3 = getelementptr inbounds <8 x i16>, ptr %0, i64 1
%4 = load <8 x i16>, ptr %3, align 1
@@ -912,6 +1257,17 @@ define void @store_factor2(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
; CHECK-IADISABLED-NEXT: zip1 v0.4s, v2.4s, v0.4s
; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2:
+; CHECK-GI: .Lfunc_begin11:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: trn1 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: zip1 v1.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -932,6 +1288,20 @@ define void @store_factor2_high(ptr %ptr, ptr %ptr2, <4 x i32> %a0, <4 x i32> %a
; CHECK-NEXT: str q1, [x0]
; CHECK-NEXT: str q0, [x1]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_high:
+; CHECK-GI: .Lfunc_begin12:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: adrp x8, .LCPI12_0
+; CHECK-GI-NEXT: trn1 v3.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI12_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b
+; CHECK-GI-NEXT: zip2 v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: str q0, [x0]
+; CHECK-GI-NEXT: str q1, [x1]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 0, i32 4, i32 1, i32 6>
@@ -952,6 +1322,20 @@ define void @store_factor2_high2(ptr %ptr, ptr %ptr2, <4 x i32> %a0, <4 x i32> %
; CHECK-NEXT: str q2, [x0]
; CHECK-NEXT: str q0, [x1]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_high2:
+; CHECK-GI: .Lfunc_begin13:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI13_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q2, [x0]
+; CHECK-GI-NEXT: str q0, [x1]
+; CHECK-GI-NEXT: ret
%interleaved.vec = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 6>
%interleaved.vec2 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
store <4 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -992,6 +1376,35 @@ define void @store_factor3(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2
; CHECK-IADISABLED-NEXT: ext v0.16b, v5.16b, v3.16b, #4
; CHECK-IADISABLED-NEXT: stp q0, q4, [x0, #16]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3:
+; CHECK-GI: .Lfunc_begin14:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: adrp x8, .LCPI14_0
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-GI-NEXT: tbl v5.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-GI-NEXT: tbl v6.16b, { v1.16b, v2.16b }, v4.16b
+; CHECK-GI-NEXT: mov v3.16b, v0.16b
+; CHECK-GI-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v4.16b
+; CHECK-GI-NEXT: mov s1, v5.s[0]
+; CHECK-GI-NEXT: mov s2, v6.s[1]
+; CHECK-GI-NEXT: mov s3, v0.s[2]
+; CHECK-GI-NEXT: mov v1.s[1], v6.s[0]
+; CHECK-GI-NEXT: mov v2.s[1], v0.s[1]
+; CHECK-GI-NEXT: mov v3.s[1], v5.s[3]
+; CHECK-GI-NEXT: mov v1.s[2], v0.s[0]
+; CHECK-GI-NEXT: mov v2.s[2], v5.s[2]
+; CHECK-GI-NEXT: mov v3.s[2], v6.s[3]
+; CHECK-GI-NEXT: mov v1.s[3], v5.s[1]
+; CHECK-GI-NEXT: mov v2.s[3], v6.s[2]
+; CHECK-GI-NEXT: mov v3.s[3], v0.s[3]
+; CHECK-GI-NEXT: stp q1, q2, [x0]
+; CHECK-GI-NEXT: str q3, [x0, #32]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a0, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
@@ -1035,6 +1448,34 @@ define void @store_factor4(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2
; CHECK-IADISABLED-NEXT: stp q17, q7, [x0, #32]
; CHECK-IADISABLED-NEXT: stp q0, q4, [x0]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4:
+; CHECK-GI: .Lfunc_begin15:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: trn1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: trn1 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: trn1 v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: mov s5, v4.s[0]
+; CHECK-GI-NEXT: mov s6, v4.s[1]
+; CHECK-GI-NEXT: mov s7, v4.s[2]
+; CHECK-GI-NEXT: mov s4, v4.s[3]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v6.s[1], v1.s[1]
+; CHECK-GI-NEXT: mov v7.s[1], v1.s[2]
+; CHECK-GI-NEXT: mov v4.s[1], v1.s[3]
+; CHECK-GI-NEXT: mov v5.s[2], v2.s[0]
+; CHECK-GI-NEXT: mov v6.s[2], v2.s[1]
+; CHECK-GI-NEXT: mov v7.s[2], v2.s[2]
+; CHECK-GI-NEXT: mov v4.s[2], v2.s[3]
+; CHECK-GI-NEXT: mov v5.s[3], v0.s[0]
+; CHECK-GI-NEXT: mov v6.s[3], v0.s[1]
+; CHECK-GI-NEXT: mov v7.s[3], v0.s[2]
+; CHECK-GI-NEXT: mov v4.s[3], v0.s[3]
+; CHECK-GI-NEXT: stp q5, q6, [x0]
+; CHECK-GI-NEXT: stp q7, q4, [x0, #32]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a3, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -1056,6 +1497,16 @@ define void @debuginfo(ptr nocapture noundef writeonly %buf, <8 x i16> noundef %
; CHECK-NEXT: zip2 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: stp q2, q0, [x0]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: debuginfo:
+; CHECK-GI: .Lfunc_begin16:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: zip1 v2.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: zip2 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: stp q2, q0, [x0]
+; CHECK-GI-NEXT: ret
entry:
%vzip.i = shufflevector <8 x i16> %a, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 poison, i16 poison>, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
%vzip1.i = shufflevector <8 x i16> %a, <8 x i16> <i16 poison, i16 poison, i16 poison, i16 poison, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
@@ -1086,6 +1537,25 @@ define void @vld2_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture
; CHECK-NEXT: b.ne .LBB17_1
; CHECK-NEXT: // %bb.2: // %while.end
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2_intrinsic:
+; CHECK-GI: .Lfunc_begin17:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB17_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0], #32
+; CHECK-GI-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: fmul v2.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v2.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: str q2, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB17_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1130,6 +1600,24 @@ define void @vld3_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture
; CHECK-NEXT: b.ne .LBB18_1
; CHECK-NEXT: // %bb.2: // %while.end
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3_intrinsic:
+; CHECK-GI: .Lfunc_begin18:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB18_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-GI-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: str q3, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB18_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1202,6 +1690,28 @@ define void @vld4_intrinsic(ptr nocapture readonly %pSrc, ptr noalias nocapture
; CHECK-IADISABLED-NEXT: b.ne .LBB19_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4_intrinsic:
+; CHECK-GI: .Lfunc_begin19:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB19_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-GI-NEXT: add x9, x1, x8
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: zip1 v1.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: stp q1, q0, [x9]
+; CHECK-GI-NEXT: b.ne .LBB19_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1261,6 +1771,30 @@ define void @twosrc_intrinsic(ptr nocapture readonly %pSrc, ptr nocapture readon
; CHECK-NEXT: b.ne .LBB20_1
; CHECK-NEXT: // %bb.2: // %while.end
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: twosrc_intrinsic:
+; CHECK-GI: .Lfunc_begin20:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB20_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: add x9, x0, x8
+; CHECK-GI-NEXT: add x10, x1, x8
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: ldp q0, q1, [x9]
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: ldp q2, q3, [x10]
+; CHECK-GI-NEXT: uzp1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v5.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: uzp2 v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: fmul v4.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: fmla v4.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: str q4, [x2], #16
+; CHECK-GI-NEXT: b.ne .LBB20_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1330,6 +1864,25 @@ define void @vld2_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias n
; CHECK-IADISABLED-NEXT: b.ne .LBB21_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld2_multiuse_intrinsic:
+; CHECK-GI: .Lfunc_begin21:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB21_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ldp q0, q1, [x0], #32
+; CHECK-GI-NEXT: uzp1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: fmul v1.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v1.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: str q1, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB21_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1373,6 +1926,24 @@ define void @vld3_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias n
; CHECK-NEXT: b.ne .LBB22_1
; CHECK-NEXT: // %bb.2: // %while.end
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: vld3_multiuse_intrinsic:
+; CHECK-GI: .Lfunc_begin22:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB22_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+; CHECK-GI-NEXT: fmul v3.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmla v3.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmla v3.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: str q3, [x1, x8]
+; CHECK-GI-NEXT: add x8, x8, #16
+; CHECK-GI-NEXT: cmp x8, #1, lsl #12 // =4096
+; CHECK-GI-NEXT: b.ne .LBB22_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1443,6 +2014,28 @@ define void @vld4_multiuse_intrinsic(ptr nocapture readonly %pSrc, ptr noalias n
; CHECK-IADISABLED-NEXT: b.ne .LBB23_1
; CHECK-IADISABLED-NEXT: // %bb.2: // %while.end
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: vld4_multiuse_intrinsic:
+; CHECK-GI: .Lfunc_begin23:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, xzr
+; CHECK-GI-NEXT: .LBB23_1: // %vector.body
+; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x0], #64
+; CHECK-GI-NEXT: add x9, x1, x8
+; CHECK-GI-NEXT: add x8, x8, #32
+; CHECK-GI-NEXT: cmp x8, #2, lsl #12 // =8192
+; CHECK-GI-NEXT: fmul v4.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: fmul v5.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: fmla v4.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: fmla v5.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: zip1 v1.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: stp q1, q0, [x9]
+; CHECK-GI-NEXT: b.ne .LBB23_1
+; CHECK-GI-NEXT: // %bb.2: // %while.end
+; CHECK-GI-NEXT: ret
entry:
br label %vector.body
@@ -1494,6 +2087,26 @@ define void @transpose_s16_8x8_simpler_intrinsic(ptr nocapture noundef %a) {
; CHECK-NEXT: str q2, [x0]
; CHECK-NEXT: str q0, [x0, #64]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_simpler_intrinsic:
+; CHECK-GI: .Lfunc_begin24:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-GI-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-GI-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-GI-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v2.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn1 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q2, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #64]
+; CHECK-GI-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -1548,6 +2161,30 @@ define void @transpose_s16_8x8_simpler2_intrinsic(ptr nocapture noundef %a) {
; CHECK-NEXT: str q1, [x0]
; CHECK-NEXT: str q0, [x0, #64]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_simpler2_intrinsic:
+; CHECK-GI: .Lfunc_begin25:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x0]
+; CHECK-GI-NEXT: adrp x8, .LCPI25_1
+; CHECK-GI-NEXT: ldp q2, q3, [x0, #32]
+; CHECK-GI-NEXT: adrp x9, .LCPI25_0
+; CHECK-GI-NEXT: ldp q4, q5, [x0, #64]
+; CHECK-GI-NEXT: ldp q6, q7, [x0, #96]
+; CHECK-GI-NEXT: trn1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn1 v2.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v1.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI25_1]
+; CHECK-GI-NEXT: ldr q5, [x9, :lo12:.LCPI25_0]
+; CHECK-GI-NEXT: trn1 v3.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v4.16b
+; CHECK-GI-NEXT: tbl v1.16b, { v2.16b, v3.16b }, v5.16b
+; CHECK-GI-NEXT: zip1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: str q2, [x0]
+; CHECK-GI-NEXT: str q0, [x0, #64]
+; CHECK-GI-NEXT: ret
entry:
%0 = load <8 x i16>, ptr %a, align 16
%arrayidx1 = getelementptr inbounds <8 x i16>, ptr %a, i64 1
@@ -1628,6 +2265,52 @@ define void @transpose_s16_8x8_intrinsic(ptr nocapture noundef %0, ptr nocapture
; CHECK-NEXT: str q2, [x6]
; CHECK-NEXT: str q0, [x7]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: transpose_s16_8x8_intrinsic:
+; CHECK-GI: .Lfunc_begin26:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: ldr q1, [x1]
+; CHECK-GI-NEXT: ldr q2, [x2]
+; CHECK-GI-NEXT: ldr q3, [x3]
+; CHECK-GI-NEXT: ldr q4, [x4]
+; CHECK-GI-NEXT: ldr q5, [x5]
+; CHECK-GI-NEXT: trn1 v16.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: trn2 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: ldr q6, [x6]
+; CHECK-GI-NEXT: ldr q7, [x7]
+; CHECK-GI-NEXT: trn1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn1 v17.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn2 v2.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: trn2 v3.8h, v4.8h, v5.8h
+; CHECK-GI-NEXT: trn1 v18.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn2 v4.8h, v6.8h, v7.8h
+; CHECK-GI-NEXT: trn1 v5.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn2 v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: trn1 v7.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn2 v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: trn1 v6.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn1 v19.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: trn2 v1.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: trn2 v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: zip1 v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip1 v4.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: zip1 v17.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip1 v18.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: zip2 v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: zip2 v1.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: str q3, [x0]
+; CHECK-GI-NEXT: zip2 v3.4s, v7.4s, v19.4s
+; CHECK-GI-NEXT: str q4, [x1]
+; CHECK-GI-NEXT: str q17, [x2]
+; CHECK-GI-NEXT: str q18, [x3]
+; CHECK-GI-NEXT: str q5, [x4]
+; CHECK-GI-NEXT: str q3, [x5]
+; CHECK-GI-NEXT: str q1, [x6]
+; CHECK-GI-NEXT: str q0, [x7]
+; CHECK-GI-NEXT: ret
%9 = load <8 x i16>, ptr %0, align 16
%10 = load <8 x i16>, ptr %1, align 16
%11 = shufflevector <8 x i16> %9, <8 x i16> %10, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
@@ -1704,6 +2387,17 @@ define void @store_factor2_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1) {
; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v2.4s, v0.4s
; CHECK-IADISABLED-NEXT: stp q0, q1, [x0]
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor2_intrinsic:
+; CHECK-GI: .Lfunc_begin27:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: trn1 v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: trn1 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: zip1 v1.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: zip2 v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: stp q1, q0, [x0]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%interleaved.vec = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %v0, <4 x i32> %v1)
@@ -1727,6 +2421,22 @@ define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4
; CHECK-NEXT: mov v5.s[0], v2.s[0]
; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor3_intrinsic:
+; CHECK-GI: .Lfunc_begin28:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: ext v3.16b, v0.16b, v1.16b, #12
+; CHECK-GI-NEXT: ext v6.16b, v1.16b, v2.16b, #12
+; CHECK-GI-NEXT: zip2 v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: mov v3.s[0], v0.s[0]
+; CHECK-GI-NEXT: ext v0.16b, v2.16b, v0.16b, #12
+; CHECK-GI-NEXT: zip2 v4.4s, v1.4s, v6.4s
+; CHECK-GI-NEXT: mov v4.s[0], v1.s[0]
+; CHECK-GI-NEXT: zip2 v5.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: mov v5.s[0], v2.s[0]
+; CHECK-GI-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a0, <4 x i32> <i32 0, i32 5, i32 3, i32 6>
@@ -1746,6 +2456,17 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %a0, <4 x i32> %a1, <4
; CHECK-NEXT: trn1 v7.4s, v3.4s, v0.4s
; CHECK-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: store_factor4_intrinsic:
+; CHECK-GI: .Lfunc_begin29:
+; CHECK-GI-NEXT: .cfi_startproc
+; CHECK-GI-NEXT: // %bb.0:
+; CHECK-GI-NEXT: trn1 v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: trn1 v5.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: trn1 v6.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: trn1 v7.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; CHECK-GI-NEXT: ret
%v0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v1 = shufflevector <4 x i32> %a1, <4 x i32> %a2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
%v2 = shufflevector <4 x i32> %a2, <4 x i32> %a3, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
diff --git a/llvm/test/CodeGen/AArch64/zext-shuffle.ll b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
index 56fe2249b6d75..8273c825c431a 100644
--- a/llvm/test/CodeGen/AArch64/zext-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/zext-shuffle.ll
@@ -1,6 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc --lower-interleaved-accesses=false -mtriple=aarch64-none-eabi -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-none-eabi --lower-interleaved-accesses=false -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-none-eabi --lower-interleaved-accesses=false -global-isel -global-isel-abort=2 -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK-GI
+
+; CHECK-GI: warning: Instruction selection used fallback path for prevent_scalable_nvcast
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for prevent_wide_nvcast
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for prevent_scalable_nvcast_mask
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for zext_load_add_intrinsic
define <2 x i64> @v2i64_02(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: v2i64_02:
@@ -9,6 +15,12 @@ define <2 x i64> @v2i64_02(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip1 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_02:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 0, i32 2>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -21,6 +33,12 @@ define <2 x i64> @v2i64_13(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_13:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 1, i32 3>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -32,6 +50,12 @@ define <2 x i64> @v2i64_04812(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip1 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_04812:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 0, i32 4>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -43,6 +67,12 @@ define <2 x i64> @v2i64_15913(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_15913:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: trn2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 1, i32 5>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -56,6 +86,12 @@ define <2 x i64> @v2i64_261014(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip1 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_261014:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip2 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 2, i32 6>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -69,6 +105,16 @@ define <2 x i64> @v2i64_37(<4 x i32> %a, <4 x i32> %b) {
; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_37:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI5_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI5_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <4 x i32> %a, <4 x i32> %b, <2 x i32> <i32 3, i32 7>
%d = zext <2 x i32> %c to <2 x i64>
ret <2 x i64> %d
@@ -83,6 +129,18 @@ define <4 x i64> @v2i64_i16_04812(<16 x i16> %a) {
; CHECK-NEXT: ushll2 v1.2d, v0.4s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_i16_04812:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI6_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <16 x i16> %a, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%z1 = zext <4 x i16> %s1 to <4 x i64>
ret <4 x i64> %z1
@@ -97,6 +155,18 @@ define <4 x i64> @v2i64_i16_15913(<16 x i16> %a) {
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_i16_15913:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI7_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI7_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <16 x i16> %a, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%z1 = zext <4 x i16> %s1 to <4 x i64>
ret <4 x i64> %z1
@@ -111,6 +181,18 @@ define <4 x i64> @v2i64_i16_261014(<16 x i16> %a) {
; CHECK-NEXT: ushll2 v1.2d, v0.4s, #0
; CHECK-NEXT: ushll v0.2d, v0.2s, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_i16_261014:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI8_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI8_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <16 x i16> %a, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%z1 = zext <4 x i16> %s1 to <4 x i64>
ret <4 x i64> %z1
@@ -122,6 +204,18 @@ define <4 x i64> @v2i64_i16_371115(<16 x i16> %a) {
; CHECK-NEXT: ushr v0.2d, v0.2d, #48
; CHECK-NEXT: ushr v1.2d, v1.2d, #48
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v2i64_i16_371115:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI9_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI9_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <16 x i16> %a, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
%z1 = zext <4 x i16> %s1 to <4 x i64>
ret <4 x i64> %z1
@@ -134,6 +228,12 @@ define <4 x i32> @v4i32_0246(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: movi v1.2d, #0x00ffff0000ffff
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_0246:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v0.8h
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -144,6 +244,12 @@ define <4 x i32> @v4i32_1357(<8 x i16> %a, <8 x i16> %b) {
; CHECK: // %bb.0:
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_1357:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp2 v0.8h, v0.8h, v0.8h
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -156,6 +262,16 @@ define <4 x i32> @v4i32_04812(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_04812:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI12_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI12_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -167,6 +283,16 @@ define <4 x i32> @v4i32_15913(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_15913:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI13_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -179,6 +305,16 @@ define <4 x i32> @v4i32_261014(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_261014:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI14_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI14_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -190,6 +326,16 @@ define <4 x i32> @v4i32_371115(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v4i32_371115:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI15_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI15_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <8 x i16> %a, <8 x i16> %b, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
%d = zext <4 x i16> %c to <4 x i32>
ret <4 x i32> %d
@@ -201,6 +347,12 @@ define <8 x i16> @v8i16_0246(<16 x i8> %a, <16 x i8> %b) {
; CHECK: // %bb.0:
; CHECK-NEXT: bic v0.8h, #255, lsl #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_0246:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -211,6 +363,12 @@ define <8 x i16> @v8i16_1357(<16 x i8> %a, <16 x i8> %b) {
; CHECK: // %bb.0:
; CHECK-NEXT: ushr v0.8h, v0.8h, #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_1357:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -222,6 +380,16 @@ define <8 x i16> @v8i16_04812(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: bic v0.8h, #255, lsl #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_04812:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI18_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI18_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -233,6 +401,16 @@ define <8 x i16> @v8i16_15913(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_15913:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI19_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI19_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -244,6 +422,16 @@ define <8 x i16> @v8i16_261014(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: bic v0.8h, #255, lsl #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_261014:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI20_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI20_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -255,6 +443,16 @@ define <8 x i16> @v8i16_371115(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i16_371115:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI21_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI21_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
%d = zext <8 x i8> %c to <8 x i16>
ret <8 x i16> %d
@@ -268,6 +466,14 @@ define <8 x i32> @v8i32_0246(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_0246:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -280,6 +486,14 @@ define <8 x i32> @v8i32_1357(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_1357:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: uzp2 v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -292,6 +506,18 @@ define <8 x i32> @v8i32_04812(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_04812:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI24_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI24_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -306,6 +532,18 @@ define <8 x i32> @v8i32_15913(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_15913:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI25_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI25_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -319,6 +557,18 @@ define <8 x i32> @v8i32_261014(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: bic v0.4s, #255, lsl #8
; CHECK-NEXT: bic v1.4s, #255, lsl #8
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_261014:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI26_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI26_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -330,6 +580,18 @@ define <8 x i32> @v8i32_371115(<16 x i8> %a, <16 x i8> %b) {
; CHECK-NEXT: ushr v0.4s, v0.4s, #24
; CHECK-NEXT: ushr v1.4s, v1.4s, #24
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: v8i32_371115:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI27_0
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ret
%c = shufflevector <16 x i8> %a, <16 x i8> %b, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
%d = zext <8 x i8> %c to <8 x i32>
ret <8 x i32> %d
@@ -352,6 +614,50 @@ define <8 x i64> @zext_add(<32 x i16> %l) {
; CHECK-NEXT: uaddl2 v3.2d, v5.4s, v2.4s
; CHECK-NEXT: uaddl v2.2d, v5.2s, v2.2s
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: zext_add:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v4.h[0], v0.h[0]
+; CHECK-GI-NEXT: mov v5.h[0], v2.h[0]
+; CHECK-GI-NEXT: mov v6.h[0], v0.h[1]
+; CHECK-GI-NEXT: mov v7.h[0], v0.h[2]
+; CHECK-GI-NEXT: mov v16.h[0], v0.h[3]
+; CHECK-GI-NEXT: mov v17.h[0], v2.h[1]
+; CHECK-GI-NEXT: mov v18.h[0], v2.h[2]
+; CHECK-GI-NEXT: mov v19.h[0], v2.h[3]
+; CHECK-GI-NEXT: mov v4.h[1], v0.h[4]
+; CHECK-GI-NEXT: mov v5.h[1], v2.h[4]
+; CHECK-GI-NEXT: mov v6.h[1], v0.h[5]
+; CHECK-GI-NEXT: mov v7.h[1], v0.h[6]
+; CHECK-GI-NEXT: mov v16.h[1], v0.h[7]
+; CHECK-GI-NEXT: mov v17.h[1], v2.h[5]
+; CHECK-GI-NEXT: mov v18.h[1], v2.h[6]
+; CHECK-GI-NEXT: mov v19.h[1], v2.h[7]
+; CHECK-GI-NEXT: mov v4.h[2], v1.h[0]
+; CHECK-GI-NEXT: mov v5.h[2], v3.h[0]
+; CHECK-GI-NEXT: mov v6.h[2], v1.h[1]
+; CHECK-GI-NEXT: mov v7.h[2], v1.h[2]
+; CHECK-GI-NEXT: mov v16.h[2], v1.h[3]
+; CHECK-GI-NEXT: mov v17.h[2], v3.h[1]
+; CHECK-GI-NEXT: mov v18.h[2], v3.h[2]
+; CHECK-GI-NEXT: mov v19.h[2], v3.h[3]
+; CHECK-GI-NEXT: mov v4.h[3], v1.h[4]
+; CHECK-GI-NEXT: mov v5.h[3], v3.h[4]
+; CHECK-GI-NEXT: mov v6.h[3], v1.h[5]
+; CHECK-GI-NEXT: mov v7.h[3], v1.h[6]
+; CHECK-GI-NEXT: mov v16.h[3], v1.h[7]
+; CHECK-GI-NEXT: mov v17.h[3], v3.h[5]
+; CHECK-GI-NEXT: mov v18.h[3], v3.h[6]
+; CHECK-GI-NEXT: mov v19.h[3], v3.h[7]
+; CHECK-GI-NEXT: uaddl v1.4s, v4.4h, v6.4h
+; CHECK-GI-NEXT: uaddl v2.4s, v7.4h, v16.4h
+; CHECK-GI-NEXT: uaddl v3.4s, v5.4h, v17.4h
+; CHECK-GI-NEXT: uaddl v4.4s, v18.4h, v19.4h
+; CHECK-GI-NEXT: uaddl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT: uaddl2 v1.2d, v1.4s, v2.4s
+; CHECK-GI-NEXT: uaddl v2.2d, v3.2s, v4.2s
+; CHECK-GI-NEXT: uaddl2 v3.2d, v3.4s, v4.4s
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%z1 = zext <8 x i16> %s1 to <8 x i64>
%s2 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
@@ -397,6 +703,53 @@ define <8 x i64> @zext_load_add(ptr %p) {
; CHECK-IADISABLED-NEXT: uaddl2 v3.2d, v5.4s, v2.4s
; CHECK-IADISABLED-NEXT: uaddl v2.2d, v5.2s, v2.2s
; CHECK-IADISABLED-NEXT: ret
+;
+; CHECK-GI-LABEL: zext_load_add:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q2, q4, [x0, #16]
+; CHECK-GI-NEXT: ldr q5, [x0]
+; CHECK-GI-NEXT: ldr q1, [x0, #48]
+; CHECK-GI-NEXT: mov v0.h[0], v5.h[0]
+; CHECK-GI-NEXT: mov v6.h[0], v5.h[1]
+; CHECK-GI-NEXT: mov v7.h[0], v5.h[2]
+; CHECK-GI-NEXT: mov v3.h[0], v4.h[0]
+; CHECK-GI-NEXT: mov v16.h[0], v5.h[3]
+; CHECK-GI-NEXT: mov v17.h[0], v4.h[1]
+; CHECK-GI-NEXT: mov v18.h[0], v4.h[2]
+; CHECK-GI-NEXT: mov v19.h[0], v4.h[3]
+; CHECK-GI-NEXT: mov v0.h[1], v5.h[4]
+; CHECK-GI-NEXT: mov v6.h[1], v5.h[5]
+; CHECK-GI-NEXT: mov v7.h[1], v5.h[6]
+; CHECK-GI-NEXT: mov v3.h[1], v4.h[4]
+; CHECK-GI-NEXT: mov v16.h[1], v5.h[7]
+; CHECK-GI-NEXT: mov v17.h[1], v4.h[5]
+; CHECK-GI-NEXT: mov v18.h[1], v4.h[6]
+; CHECK-GI-NEXT: mov v19.h[1], v4.h[7]
+; CHECK-GI-NEXT: mov v0.h[2], v2.h[0]
+; CHECK-GI-NEXT: mov v6.h[2], v2.h[1]
+; CHECK-GI-NEXT: mov v7.h[2], v2.h[2]
+; CHECK-GI-NEXT: mov v3.h[2], v1.h[0]
+; CHECK-GI-NEXT: mov v16.h[2], v2.h[3]
+; CHECK-GI-NEXT: mov v17.h[2], v1.h[1]
+; CHECK-GI-NEXT: mov v18.h[2], v1.h[2]
+; CHECK-GI-NEXT: mov v19.h[2], v1.h[3]
+; CHECK-GI-NEXT: mov v0.h[3], v2.h[4]
+; CHECK-GI-NEXT: mov v6.h[3], v2.h[5]
+; CHECK-GI-NEXT: mov v7.h[3], v2.h[6]
+; CHECK-GI-NEXT: mov v3.h[3], v1.h[4]
+; CHECK-GI-NEXT: mov v16.h[3], v2.h[7]
+; CHECK-GI-NEXT: mov v17.h[3], v1.h[5]
+; CHECK-GI-NEXT: mov v18.h[3], v1.h[6]
+; CHECK-GI-NEXT: mov v19.h[3], v1.h[7]
+; CHECK-GI-NEXT: uaddl v1.4s, v0.4h, v6.4h
+; CHECK-GI-NEXT: uaddl v2.4s, v7.4h, v16.4h
+; CHECK-GI-NEXT: uaddl v3.4s, v3.4h, v17.4h
+; CHECK-GI-NEXT: uaddl v4.4s, v18.4h, v19.4h
+; CHECK-GI-NEXT: uaddl v0.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT: uaddl2 v1.2d, v1.4s, v2.4s
+; CHECK-GI-NEXT: uaddl v2.2d, v3.2s, v4.2s
+; CHECK-GI-NEXT: uaddl2 v3.2d, v3.4s, v4.4s
+; CHECK-GI-NEXT: ret
%l = load <32 x i16>, ptr %p
%s1 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%z1 = zext <8 x i16> %s1 to <8 x i64>
@@ -469,6 +822,94 @@ define <8 x double> @uitofp_fadd(<32 x i16> %l) {
; CHECK-NEXT: fadd v2.2d, v7.2d, v2.2d
; CHECK-NEXT: fadd v3.2d, v17.2d, v3.2d
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: uitofp_fadd:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v4.h[0], v0.h[0]
+; CHECK-GI-NEXT: mov v5.h[0], v2.h[0]
+; CHECK-GI-NEXT: mov v6.h[0], v0.h[1]
+; CHECK-GI-NEXT: mov v7.h[0], v2.h[1]
+; CHECK-GI-NEXT: mov v16.h[0], v0.h[2]
+; CHECK-GI-NEXT: mov v17.h[0], v2.h[2]
+; CHECK-GI-NEXT: mov v18.h[0], v0.h[3]
+; CHECK-GI-NEXT: mov v19.h[0], v2.h[3]
+; CHECK-GI-NEXT: mov v4.h[1], v0.h[4]
+; CHECK-GI-NEXT: mov v5.h[1], v2.h[4]
+; CHECK-GI-NEXT: mov v6.h[1], v0.h[5]
+; CHECK-GI-NEXT: mov v7.h[1], v2.h[5]
+; CHECK-GI-NEXT: mov v16.h[1], v0.h[6]
+; CHECK-GI-NEXT: mov v17.h[1], v2.h[6]
+; CHECK-GI-NEXT: mov v18.h[1], v0.h[7]
+; CHECK-GI-NEXT: mov v19.h[1], v2.h[7]
+; CHECK-GI-NEXT: mov v4.h[2], v1.h[0]
+; CHECK-GI-NEXT: mov v5.h[2], v3.h[0]
+; CHECK-GI-NEXT: mov v6.h[2], v1.h[1]
+; CHECK-GI-NEXT: mov v7.h[2], v3.h[1]
+; CHECK-GI-NEXT: mov v16.h[2], v1.h[2]
+; CHECK-GI-NEXT: mov v17.h[2], v3.h[2]
+; CHECK-GI-NEXT: mov v18.h[2], v1.h[3]
+; CHECK-GI-NEXT: mov v19.h[2], v3.h[3]
+; CHECK-GI-NEXT: mov v4.h[3], v1.h[4]
+; CHECK-GI-NEXT: mov v5.h[3], v3.h[4]
+; CHECK-GI-NEXT: mov v6.h[3], v1.h[5]
+; CHECK-GI-NEXT: mov v7.h[3], v3.h[5]
+; CHECK-GI-NEXT: mov v16.h[3], v1.h[6]
+; CHECK-GI-NEXT: mov v17.h[3], v3.h[6]
+; CHECK-GI-NEXT: mov v18.h[3], v1.h[7]
+; CHECK-GI-NEXT: mov v19.h[3], v3.h[7]
+; CHECK-GI-NEXT: ushll v0.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll v4.4s, v6.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v16.4h, #0
+; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT: ushll v19.4s, v19.4h, #0
+; CHECK-GI-NEXT: ushll v5.2d, v0.2s, #0
+; CHECK-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-GI-NEXT: ushll v6.2d, v2.2s, #0
+; CHECK-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-GI-NEXT: ushll v7.2d, v4.2s, #0
+; CHECK-GI-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-GI-NEXT: ushll v16.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ushll v20.2d, v3.2s, #0
+; CHECK-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-GI-NEXT: ushll v21.2d, v17.2s, #0
+; CHECK-GI-NEXT: ushll2 v17.2d, v17.4s, #0
+; CHECK-GI-NEXT: ushll v22.2d, v18.2s, #0
+; CHECK-GI-NEXT: ushll2 v18.2d, v18.4s, #0
+; CHECK-GI-NEXT: ushll v23.2d, v19.2s, #0
+; CHECK-GI-NEXT: ushll2 v19.2d, v19.4s, #0
+; CHECK-GI-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-GI-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-GI-NEXT: ucvtf v22.2d, v22.2d
+; CHECK-GI-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-GI-NEXT: ucvtf v23.2d, v23.2d
+; CHECK-GI-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-GI-NEXT: fadd v5.2d, v5.2d, v7.2d
+; CHECK-GI-NEXT: fadd v4.2d, v0.2d, v4.2d
+; CHECK-GI-NEXT: fadd v6.2d, v6.2d, v16.2d
+; CHECK-GI-NEXT: fadd v7.2d, v2.2d, v1.2d
+; CHECK-GI-NEXT: fadd v0.2d, v20.2d, v22.2d
+; CHECK-GI-NEXT: fadd v1.2d, v3.2d, v18.2d
+; CHECK-GI-NEXT: fadd v2.2d, v21.2d, v23.2d
+; CHECK-GI-NEXT: fadd v3.2d, v17.2d, v19.2d
+; CHECK-GI-NEXT: fadd v0.2d, v5.2d, v0.2d
+; CHECK-GI-NEXT: fadd v1.2d, v4.2d, v1.2d
+; CHECK-GI-NEXT: fadd v2.2d, v6.2d, v2.2d
+; CHECK-GI-NEXT: fadd v3.2d, v7.2d, v3.2d
+; CHECK-GI-NEXT: ret
%s1 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%z1 = uitofp <8 x i16> %s1 to <8 x double>
%s2 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
@@ -542,6 +983,96 @@ define <8 x double> @uitofp_load_fadd(ptr %p) {
; CHECK-NEXT: fadd v2.2d, v7.2d, v3.2d
; CHECK-NEXT: fadd v3.2d, v17.2d, v4.2d
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: uitofp_load_fadd:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldp q5, q1, [x0]
+; CHECK-GI-NEXT: ldp q3, q0, [x0, #32]
+; CHECK-GI-NEXT: mov v2.h[0], v5.h[0]
+; CHECK-GI-NEXT: mov v6.h[0], v5.h[1]
+; CHECK-GI-NEXT: mov v16.h[0], v5.h[2]
+; CHECK-GI-NEXT: mov v4.h[0], v3.h[0]
+; CHECK-GI-NEXT: mov v7.h[0], v3.h[1]
+; CHECK-GI-NEXT: mov v17.h[0], v3.h[2]
+; CHECK-GI-NEXT: mov v18.h[0], v5.h[3]
+; CHECK-GI-NEXT: mov v19.h[0], v3.h[3]
+; CHECK-GI-NEXT: mov v2.h[1], v5.h[4]
+; CHECK-GI-NEXT: mov v6.h[1], v5.h[5]
+; CHECK-GI-NEXT: mov v16.h[1], v5.h[6]
+; CHECK-GI-NEXT: mov v4.h[1], v3.h[4]
+; CHECK-GI-NEXT: mov v7.h[1], v3.h[5]
+; CHECK-GI-NEXT: mov v17.h[1], v3.h[6]
+; CHECK-GI-NEXT: mov v18.h[1], v5.h[7]
+; CHECK-GI-NEXT: mov v19.h[1], v3.h[7]
+; CHECK-GI-NEXT: mov v2.h[2], v1.h[0]
+; CHECK-GI-NEXT: mov v6.h[2], v1.h[1]
+; CHECK-GI-NEXT: mov v16.h[2], v1.h[2]
+; CHECK-GI-NEXT: mov v4.h[2], v0.h[0]
+; CHECK-GI-NEXT: mov v7.h[2], v0.h[1]
+; CHECK-GI-NEXT: mov v17.h[2], v0.h[2]
+; CHECK-GI-NEXT: mov v18.h[2], v1.h[3]
+; CHECK-GI-NEXT: mov v19.h[2], v0.h[3]
+; CHECK-GI-NEXT: mov v2.h[3], v1.h[4]
+; CHECK-GI-NEXT: mov v6.h[3], v1.h[5]
+; CHECK-GI-NEXT: mov v16.h[3], v1.h[6]
+; CHECK-GI-NEXT: mov v4.h[3], v0.h[4]
+; CHECK-GI-NEXT: mov v7.h[3], v0.h[5]
+; CHECK-GI-NEXT: mov v17.h[3], v0.h[6]
+; CHECK-GI-NEXT: mov v18.h[3], v1.h[7]
+; CHECK-GI-NEXT: mov v19.h[3], v0.h[7]
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll v4.4s, v16.4h, #0
+; CHECK-GI-NEXT: ushll v5.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll v16.4s, v18.4h, #0
+; CHECK-GI-NEXT: ushll v19.4s, v19.4h, #0
+; CHECK-GI-NEXT: ushll v6.2d, v2.2s, #0
+; CHECK-GI-NEXT: ushll2 v2.2d, v2.4s, #0
+; CHECK-GI-NEXT: ushll v17.2d, v0.2s, #0
+; CHECK-GI-NEXT: ushll v7.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-GI-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-GI-NEXT: ushll v18.2d, v3.2s, #0
+; CHECK-GI-NEXT: ushll2 v3.2d, v3.4s, #0
+; CHECK-GI-NEXT: ushll v20.2d, v4.2s, #0
+; CHECK-GI-NEXT: ushll2 v4.2d, v4.4s, #0
+; CHECK-GI-NEXT: ushll v21.2d, v5.2s, #0
+; CHECK-GI-NEXT: ushll2 v5.2d, v5.4s, #0
+; CHECK-GI-NEXT: ushll v22.2d, v16.2s, #0
+; CHECK-GI-NEXT: ushll2 v16.2d, v16.4s, #0
+; CHECK-GI-NEXT: ushll v23.2d, v19.2s, #0
+; CHECK-GI-NEXT: ushll2 v19.2d, v19.4s, #0
+; CHECK-GI-NEXT: ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT: ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT: ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT: ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT: ucvtf v17.2d, v17.2d
+; CHECK-GI-NEXT: ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT: ucvtf v18.2d, v18.2d
+; CHECK-GI-NEXT: ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT: ucvtf v20.2d, v20.2d
+; CHECK-GI-NEXT: ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT: ucvtf v21.2d, v21.2d
+; CHECK-GI-NEXT: ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT: ucvtf v22.2d, v22.2d
+; CHECK-GI-NEXT: ucvtf v16.2d, v16.2d
+; CHECK-GI-NEXT: ucvtf v23.2d, v23.2d
+; CHECK-GI-NEXT: ucvtf v19.2d, v19.2d
+; CHECK-GI-NEXT: fadd v6.2d, v6.2d, v17.2d
+; CHECK-GI-NEXT: fadd v2.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT: fadd v7.2d, v7.2d, v18.2d
+; CHECK-GI-NEXT: fadd v3.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT: fadd v0.2d, v20.2d, v22.2d
+; CHECK-GI-NEXT: fadd v1.2d, v4.2d, v16.2d
+; CHECK-GI-NEXT: fadd v4.2d, v21.2d, v23.2d
+; CHECK-GI-NEXT: fadd v5.2d, v5.2d, v19.2d
+; CHECK-GI-NEXT: fadd v0.2d, v6.2d, v0.2d
+; CHECK-GI-NEXT: fadd v1.2d, v2.2d, v1.2d
+; CHECK-GI-NEXT: fadd v2.2d, v7.2d, v4.2d
+; CHECK-GI-NEXT: fadd v3.2d, v3.2d, v5.2d
+; CHECK-GI-NEXT: ret
%l = load <32 x i16>, ptr %p
%s1 = shufflevector <32 x i16> %l, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
%z1 = uitofp <8 x i16> %s1 to <8 x double>
@@ -564,6 +1095,14 @@ define <4 x i32> @isUndefDeInterleave_b0(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_b0:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI32_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI32_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -576,6 +1115,14 @@ define <4 x i32> @isUndefDeInterleave_b1(<8 x i16> %a) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_b1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI33_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI33_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 poison, i32 poison>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -589,6 +1136,12 @@ define <4 x i32> @isUndefDeInterleave_b2(<8 x i16> %a) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_b2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v0.h[3], v0.h[6]
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -601,6 +1154,14 @@ define <4 x i32> @isUndefDeInterleave_b3(<8 x i16> %a) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_b3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI35_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI35_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -614,6 +1175,14 @@ define <4 x i32> @isUndefDeInterleave_t0(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_t0:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI36_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI36_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 4>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -626,6 +1195,14 @@ define <4 x i32> @isUndefDeInterleave_t1(<8 x i16> %a) {
; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_t1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI37_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI37_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 5>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -639,6 +1216,14 @@ define <4 x i32> @isUndefDeInterleave_t2(<8 x i16> %a) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_t2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI38_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -651,6 +1236,12 @@ define <4 x i32> @isUndefDeInterleave_t3(<8 x i16> %a) {
; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ushr v0.4s, v0.4s, #16
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_t3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov v0.h[6], v0.h[3]
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 3, i32 7>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -665,6 +1256,14 @@ define <4 x i32> @isUndefDeInterleave_b0_bad(<8 x i16> %a, <8 x i16> %b) {
; CHECK-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_b0_bad:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI40_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI40_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 4, i32 4, i32 0, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -679,6 +1278,14 @@ define <4 x i32> @isUndefDeInterleave_t1_bad(<8 x i16> %a) {
; CHECK-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: isUndefDeInterleave_t1_bad:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI41_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI41_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 4, i32 1, i32 5>
%s2 = shufflevector <8 x i16> %2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = zext <4 x i16> %s2 to <4 x i32>
@@ -691,6 +1298,17 @@ define i16 @undeftop(<8 x i16> %0) {
; CHECK-NEXT: add v0.8h, v0.8h, v0.8h
; CHECK-NEXT: umov w0, v0.h[4]
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: undeftop:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI42_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI42_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: uaddl v1.4s, v0.4h, v0.4h
+; CHECK-GI-NEXT: uaddl2 v0.4s, v0.8h, v0.8h
+; CHECK-GI-NEXT: uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT: umov w0, v0.h[0]
+; CHECK-GI-NEXT: ret
%2 = shufflevector <8 x i16> %0, <8 x i16> zeroinitializer, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 9, i32 7, i32 5, i32 3>
%3 = zext <8 x i16> %2 to <8 x i64>
%new0 = add <8 x i64> %3, %3
@@ -707,6 +1325,12 @@ define <4 x i32> @prevent_scalable_nvcast(<vscale x 8 x i16> %v) #0 {
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: prevent_scalable_nvcast:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
entry:
%v.even = call <vscale x 8 x i16> @llvm.aarch64.sve.uzp1.nxv8i16(<vscale x 8 x i16> %v,
<vscale x 8 x i16> %v)
@@ -725,6 +1349,12 @@ define <4 x i32> @prevent_wide_nvcast(<vscale x 8 x i16> %v) #1 {
; CHECK-NEXT: uzp2 z0.h, z0.h, z0.h
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: prevent_wide_nvcast:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uzp2 z0.h, z0.h, z0.h
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
entry:
%v.fixed = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> %v, i64 0)
%v.odd = shufflevector <16 x i16> %v.fixed, <16 x i16> %v.fixed, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15,
@@ -744,6 +1374,13 @@ define <4 x i32> @prevent_scalable_nvcast_mask(<vscale x 8 x i16> %v) #0 {
; CHECK-NEXT: bic v0.4h, #255, lsl #8
; CHECK-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-NEXT: ret
+;
+; CHECK-GI-LABEL: prevent_scalable_nvcast_mask:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-GI-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ret
entry:
%v.even = call <vscale x 8 x i16> @llvm.aarch64.sve.uzp1.nxv8i16(<vscale x 8 x i16> %v,
<vscale x 8 x i16> %v)
@@ -766,20 +1403,33 @@ define <8 x i64> @zext_load_add_intrinsic(ptr %p) {
; CHECK-NEXT: uaddl2 v3.2d, v6.4s, v2.4s
; CHECK-NEXT: uaddl v2.2d, v6.2s, v2.2s
; CHECK-NEXT: ret
- %l = load <32 x i16>, ptr %p
- %deint = call {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} @llvm.vector.deinterleave4(<32 x i16>%l)
- %s1 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 0
- %z1 = zext <8 x i16> %s1 to <8 x i64>
- %s2 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 1
- %z2 = zext <8 x i16> %s2 to <8 x i64>
- %s3 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 2
- %z3 = zext <8 x i16> %s3 to <8 x i64>
- %s4 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 3
- %z4 = zext <8 x i16> %s4 to <8 x i64>
- %a = add <8 x i64> %z1, %z2
- %b = add <8 x i64> %z3, %z4
- %c = add <8 x i64> %a, %b
- ret <8 x i64> %c
+;
+; CHECK-GI-LABEL: zext_load_add_intrinsic:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ld4 { v0.8h, v1.8h, v2.8h, v3.8h }, [x0]
+; CHECK-GI-NEXT: uaddl v4.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: uaddl v5.4s, v2.4h, v3.4h
+; CHECK-GI-NEXT: uaddl2 v6.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: uaddl2 v2.4s, v2.8h, v3.8h
+; CHECK-GI-NEXT: uaddl v0.2d, v4.2s, v5.2s
+; CHECK-GI-NEXT: uaddl2 v1.2d, v4.4s, v5.4s
+; CHECK-GI-NEXT: uaddl2 v3.2d, v6.4s, v2.4s
+; CHECK-GI-NEXT: uaddl v2.2d, v6.2s, v2.2s
+; CHECK-GI-NEXT: ret
+ %l = load <32 x i16>, ptr %p
+ %deint = call {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} @llvm.vector.deinterleave4(<32 x i16>%l)
+ %s1 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 0
+ %z1 = zext <8 x i16> %s1 to <8 x i64>
+ %s2 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 1
+ %z2 = zext <8 x i16> %s2 to <8 x i64>
+ %s3 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 2
+ %z3 = zext <8 x i16> %s3 to <8 x i64>
+ %s4 = extractvalue {<8 x i16>, <8 x i16>, <8 x i16>, <8 x i16>} %deint, 3
+ %z4 = zext <8 x i16> %s4 to <8 x i64>
+ %a = add <8 x i64> %z1, %z2
+ %b = add <8 x i64> %z3, %z4
+ %c = add <8 x i64> %a, %b
+ ret <8 x i64> %c
}
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
More information about the llvm-commits
mailing list