[llvm-branch-commits] [llvm] [AArch64][DAG] Improve codegen for clearing upper vector lanes (PR #237259)
Kieran B via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 02:11:12 PST 2026
https://github.com/kieroxide updated https://github.com/llvm/llvm-project/pull/237259
>From 4cf0cbfd8f16ddfbc212cf0d50411bd7585f6025 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:15 +0000
Subject: [PATCH 1/2] [AArch64][DAG] Improve codegen for clearing upper vector
lanes
Add patterns that use scalar fmov instruction to clear the upper lanes of a vector
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 69 ++++
.../CodeGen/AArch64/shuffle-insert-zero.ll | 321 +++++++++++++++++-
2 files changed, 374 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 4cc709757a8413..21066063805ad2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8769,6 +8769,75 @@ defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v4i16, scalar_to_vector, INSvi8lane
defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v2i32, scalar_to_vector_v2f32, INSvi8lane, DUPi8, bsub, VecIndex_x4>;
defm : Neon_INS_elt_ext_pattern<v8i16, v4i16, v2i32, scalar_to_vector_v2f32, INSvi16lane, DUPi16, hsub, VecIndex_x2>;
+
+let Predicates = [HasFullFP16] in {
+ def : Pat<(v8i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract
+ (v8i16 V128:$Rn),
+ (i64 0))),
+ (i64 0))),
+ (v8i16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v8i16 V128:$Rn), hsub))),
+ hsub))>;
+
+ def : Pat<(v8f16 (vector_insert immAllZerosV,
+ (f16 (vector_extract
+ (v8f16 V128:$Rn),
+ (i64 0))),
+ (i64 0))),
+ (v8f16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v8f16 V128:$Rn), hsub))),
+ hsub))>;
+
+ def : Pat<(v8bf16 (vector_insert immAllZerosV,
+ (bf16 (vector_extract
+ (v8bf16 V128:$Rn),
+ (i64 0))),
+ (i64 0))),
+ (v8bf16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v8bf16 V128:$Rn), hsub))),
+ hsub))>;
+
+ def : Pat<(v4i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract
+ (v8i16 (insert_subvector (v8i16 undef),
+ (v4i16 V64:$Rn), (i64 0))),
+ (i64 0))),
+ (i64 0))),
+ (v4i16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v4i16 V64:$Rn), hsub))),
+ hsub))>;
+
+ def : Pat<(v4i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract
+ (v8i16 (bitconvert
+ (v8f16 (insert_subvector (v8f16 undef),
+ (v4f16 V64:$Rn), (i64 0))))),
+ (i64 0))),
+ (i64 0))),
+ (v4i16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v4f16 V64:$Rn), hsub))),
+ hsub))>;
+
+ def : Pat<(v4i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract
+ (v8i16 (bitconvert
+ (v8bf16 (insert_subvector (v8bf16 undef),
+ (v4bf16 V64:$Rn), (i64 0))))),
+ (i64 0))),
+ (i64 0))),
+ (v4i16 (SUBREG_TO_REG
+ (FMOVHr (f16 (EXTRACT_SUBREG (v4bf16 V64:$Rn), hsub))),
+ hsub))>;
+}
+
+// extract from lane 0 and insert into lane 0 of a zero vector
+// is equivalent to clearing upper bits of a register, we can use fmov
+def : Pat<(v4i32 (vector_insert immAllZerosV, (i32 (vector_extract (v4i32 V128:$Rn), (i64 0))), (i64 0))),
+ (v4i32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4i32 V128:$Rn), ssub)), ssub))>;
+def : Pat<(v4f32 (vector_insert immAllZerosV, (f32 (vector_extract (v4f32 V128:$Rn), (i64 0))), (i64 0))),
+ (v4f32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4f32 V128:$Rn), ssub)), ssub))>;
+
// bitcast of an extract
// f32 bitcast(vector_extract(v4i32 src, 0)) -> EXTRACT_SUBREG(src)
def : Pat<(f32 (bitconvert (i32 (vector_extract v16i8:$src, (i64 0))))),
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index 30bac9297abea6..07dc35d8ce6ee0 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -1,7 +1,27 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NO-FP16-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NO-FP16-GI
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI
+; The canonical form uses <i32 0, i32 poison, i32 poison, ... > for the zero vector operand
+; We use equivalent zeroinitializer for testing instead which has no functional change
+
+define <2 x double> @testv2f64_canonical(<2 x double> noundef %vec) {
+; CHECK-SD-LABEL: testv2f64_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov v0.d[1], xzr
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv2f64_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit1 = shufflevector <2 x double> %vec, <2 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
+ ret <2 x double> %vecinit1
+}
define <2 x double> @testv2f64_0(<2 x double> noundef %vec) {
; CHECK-SD-LABEL: testv2f64_0:
@@ -37,6 +57,26 @@ entry:
ret <2 x double> %vecins
}
+define <4 x float> @testv4f32_canonical(<4 x float> noundef %vec) {
+; CHECK-SD-LABEL: testv4f32_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4f32_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: dup v1.4s, v1.s[0]
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <4 x float> %vec, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
+ ret <4 x float> %vecinit3
+}
+
define <4 x float> @testv4f32_0(<4 x float> noundef %vec) {
; CHECK-LABEL: testv4f32_0:
; CHECK: // %bb.0: // %entry
@@ -69,6 +109,24 @@ entry:
ret <4 x float> %vecins
}
+define <2 x float> @testv2f32_canonical(<2 x float> noundef %vec) {
+; CHECK-SD-LABEL: testv2f32_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v0.s[1], wzr
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv2f32_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: zip1 v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <2 x float> %vec, <2 x float> zeroinitializer, <2 x i32> <i32 0, i32 2>
+ ret <2 x float> %vecinit3
+}
+
define <2 x float> @testv2f32_0(<2 x float> noundef %vec) {
; CHECK-SD-LABEL: testv2f32_0:
; CHECK-SD: // %bb.0: // %entry
@@ -107,6 +165,27 @@ entry:
ret <2 x float> %vecins
}
+define <8 x half> @testv8f16_canonical(<8 x half> noundef %vec) {
+; CHECK-SD-LABEL: testv8f16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8f16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT: adrp x8, .LCPI9_0
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI9_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <8 x half> %vec, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+ ret <8 x half> %vecinit3
+}
+
define <8 x half> @testv8f16_0(<8 x half> noundef %vec) {
; CHECK-LABEL: testv8f16_0:
; CHECK: // %bb.0: // %entry
@@ -122,17 +201,17 @@ entry:
define <8 x half> @testv8f16_1(<8 x half> noundef %vec) {
; CHECK-SD-LABEL: testv8f16_1:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: adrp x8, .LCPI7_0
-; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-SD-NEXT: adrp x8, .LCPI11_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
; CHECK-SD-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv8f16_1:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT: adrp x8, .LCPI7_0
+; CHECK-GI-NEXT: adrp x8, .LCPI11_0
; CHECK-GI-NEXT: movi d31, #0000000000000000
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
; CHECK-GI-NEXT: tbl v0.16b, { v31.16b, v0.16b }, v1.16b
; CHECK-GI-NEXT: ret
entry:
@@ -140,6 +219,28 @@ entry:
ret <8 x half> %vecins
}
+define <4 x half> @testv4f16_canonical(<4 x half> noundef %vec) {
+; CHECK-SD-LABEL: testv4f16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: fmov d0, d1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4f16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: dup v1.4h, v1.h[0]
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <4 x half> %vec, <4 x half> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
+ ret <4 x half> %vecinit3
+}
+
define <4 x half> @testv4f16_0(<4 x half> noundef %vec) {
; CHECK-SD-LABEL: testv4f16_0:
; CHECK-SD: // %bb.0: // %entry
@@ -183,6 +284,27 @@ entry:
ret <4 x half> %vecins
}
+define <8 x bfloat> @testv8bf16_canonical(<8 x bfloat> noundef %vec) {
+; CHECK-SD-LABEL: testv8bf16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8bf16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $q0 killed $q0 def $q0_q1
+; CHECK-GI-NEXT: adrp x8, .LCPI15_0
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI15_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+ ret <8 x bfloat> %vecinit3
+}
+
define <8 x bfloat> @testv8bf16_0(<8 x bfloat> noundef %vec) {
; CHECK-LABEL: testv8bf16_0:
; CHECK: // %bb.0: // %entry
@@ -198,17 +320,17 @@ entry:
define <8 x bfloat> @testv8bf16_1(<8 x bfloat> noundef %vec) {
; CHECK-SD-LABEL: testv8bf16_1:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: adrp x8, .LCPI11_0
-; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
+; CHECK-SD-NEXT: adrp x8, .LCPI17_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI17_0]
; CHECK-SD-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv8bf16_1:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT: adrp x8, .LCPI11_0
+; CHECK-GI-NEXT: adrp x8, .LCPI17_0
; CHECK-GI-NEXT: movi d31, #0000000000000000
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI17_0]
; CHECK-GI-NEXT: tbl v0.16b, { v31.16b, v0.16b }, v1.16b
; CHECK-GI-NEXT: ret
entry:
@@ -216,6 +338,28 @@ entry:
ret <8 x bfloat> %vecins
}
+define <4 x bfloat> @testv4bf16_canonical(<4 x bfloat> noundef %vec) {
+; CHECK-SD-LABEL: testv4bf16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: fmov d0, d1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4bf16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: dup v1.4h, v1.h[0]
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <4 x bfloat> %vec, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
+ ret <4 x bfloat> %vecinit3
+}
+
define <4 x bfloat> @testv4bf16_0(<4 x bfloat> noundef %vec) {
; CHECK-SD-LABEL: testv4bf16_0:
; CHECK-SD: // %bb.0: // %entry
@@ -259,6 +403,21 @@ entry:
ret <4 x bfloat> %vecins
}
+define <2 x i64> @testv2i64_canonical(<2 x i64> noundef %vec) {
+; CHECK-SD-LABEL: testv2i64_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov v0.d[1], xzr
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv2i64_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: zip1 v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit1 = shufflevector <2 x i64> %vec, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
+ ret <2 x i64> %vecinit1
+}
define <2 x i64> @testv2i64_0(<2 x i64> noundef %vec) {
; CHECK-SD-LABEL: testv2i64_0:
@@ -293,6 +452,26 @@ entry:
ret <2 x i64> %vecins
}
+define <4 x i32> @testv4i32_canonical(<4 x i32> noundef %vec) {
+; CHECK-SD-LABEL: testv4i32_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4i32_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: dup v1.4s, v1.s[0]
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <4 x i32> %vec, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
+ ret <4 x i32> %vecinit3
+}
+
define <4 x i32> @testv4i32_0(<4 x i32> noundef %vec) {
; CHECK-LABEL: testv4i32_0:
; CHECK: // %bb.0: // %entry
@@ -325,6 +504,24 @@ entry:
ret <4 x i32> %vecins
}
+define <2 x i32> @testv2i32_canonical(<2 x i32> noundef %vec) {
+; CHECK-SD-LABEL: testv2i32_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v0.s[1], wzr
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv2i32_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: zip1 v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <2 x i32> %vec, <2 x i32> zeroinitializer, <2 x i32> <i32 0, i32 2>
+ ret <2 x i32> %vecinit3
+}
+
define <2 x i32> @testv2i32_0(<2 x i32> noundef %vec) {
; CHECK-SD-LABEL: testv2i32_0:
; CHECK-SD: // %bb.0: // %entry
@@ -362,6 +559,25 @@ entry:
ret <2 x i32> %vecins
}
+define <8 x i16> @testv8i16_canonical(<8 x i16> noundef %vec) {
+; CHECK-SD-LABEL: testv8i16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8i16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: adrp x8, .LCPI30_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI30_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+ ret <8 x i16> %vecinit3
+}
+
define <8 x i16> @testv8i16_0(<8 x i16> noundef %vec) {
; CHECK-LABEL: testv8i16_0:
; CHECK: // %bb.0: // %entry
@@ -377,8 +593,8 @@ entry:
define <8 x i16> @testv8i16_1(<8 x i16> noundef %vec) {
; CHECK-LABEL: testv8i16_1:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: adrp x8, .LCPI21_0
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI21_0]
+; CHECK-NEXT: adrp x8, .LCPI32_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI32_0]
; CHECK-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-NEXT: ret
entry:
@@ -386,6 +602,28 @@ entry:
ret <8 x i16> %vecins
}
+define <4 x i16> @testv4i16_canonical(<4 x i16> noundef %vec) {
+; CHECK-SD-LABEL: testv4i16_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-SD-NEXT: fmov d0, d1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4i16_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: dup v1.4h, v1.h[0]
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
+ ret <4 x i16> %vecinit3
+}
+
define <4 x i16> @testv4i16_0(<4 x i16> noundef %vec) {
; CHECK-LABEL: testv4i16_0:
; CHECK: // %bb.0: // %entry
@@ -421,6 +659,26 @@ entry:
ret <4 x i16> %vecins
}
+define <16 x i8> @testv16i8_canonical(<16 x i8> noundef %vec) {
+; CHECK-SD-LABEL: testv16i8_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: mov v1.b[0], v0.b[0]
+; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv16i8_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: adrp x8, .LCPI36_0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI36_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <16 x i8> %vec, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+ ret <16 x i8> %vecinit3
+}
+
+
define <16 x i8> @testv16i8_0(<16 x i8> noundef %vec) {
; CHECK-LABEL: testv16i8_0:
; CHECK: // %bb.0: // %entry
@@ -436,8 +694,8 @@ entry:
define <16 x i8> @testv16i8_1(<16 x i8> noundef %vec) {
; CHECK-LABEL: testv16i8_1:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: adrp x8, .LCPI25_0
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI25_0]
+; CHECK-NEXT: adrp x8, .LCPI38_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]
; CHECK-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-NEXT: ret
entry:
@@ -445,6 +703,31 @@ entry:
ret <16 x i8> %vecins
}
+
+define <8 x i8> @testv8i8_canonical(<8 x i8> noundef %vec) {
+; CHECK-SD-LABEL: testv8i8_canonical:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mov v1.b[0], v0.b[0]
+; CHECK-SD-NEXT: fmov d0, d1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8i8_canonical:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: adrp x8, .LCPI39_0
+; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI39_0]
+; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %vecinit3 = shufflevector <8 x i8> %vec, <8 x i8> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+ ret <8 x i8> %vecinit3
+}
+
define <8 x i8> @testv8i8_0(<8 x i8> noundef %vec) {
; CHECK-LABEL: testv8i8_0:
; CHECK: // %bb.0: // %entry
@@ -471,9 +754,9 @@ define <8 x i8> @testv8i8_1(<8 x i8> noundef %vec) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: movi d1, #0000000000000000
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: adrp x8, .LCPI27_0
+; CHECK-GI-NEXT: adrp x8, .LCPI41_0
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI41_0]
; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
@@ -481,3 +764,9 @@ entry:
%vecins = shufflevector <8 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <8 x i8> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
ret <8 x i8> %vecins
}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-FP16-GI: {{.*}}
+; CHECK-FP16-SD: {{.*}}
+; CHECK-NO-FP16-GI: {{.*}}
+; CHECK-NO-FP16-SD: {{.*}}
>From 9269faebb3fdb255adcf6aefe078ec8a880fb6e8 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:32 +0000
Subject: [PATCH 2/2] Update Test Checks
---
.../aarch64-neon-vector-insert-uaddlv.ll | 30 +-
.../CodeGen/AArch64/build-vector-extract.ll | 4 +-
llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 1289 ++++++++---------
.../CodeGen/AArch64/shuffle-insert-zero.ll | 259 ++--
4 files changed, 817 insertions(+), 765 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d5523975..ad6fc44e1e7faa 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -30,8 +30,8 @@ define void @insert_vec_v4i32_uaddlv_from_v8i16(ptr %0) {
; CHECK-LABEL: insert_vec_v4i32_uaddlv_from_v8i16:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: movi.2d v0, #0000000000000000
-; CHECK-NEXT: uaddlv.8h s1, v0
-; CHECK-NEXT: mov.s v0[0], v1[0]
+; CHECK-NEXT: uaddlv.8h s0, v0
+; CHECK-NEXT: fmov s0, s0
; CHECK-NEXT: ucvtf.4s v0, v0
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
@@ -48,11 +48,10 @@ define void @insert_vec_v16i32_uaddlv_from_v8i16(ptr %0) {
; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v8i16:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: movi.2d v0, #0000000000000000
-; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.8h s1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.s v2[0], v1[0]
-; CHECK-NEXT: ucvtf.4s v1, v2
+; CHECK-NEXT: fmov s1, s1
+; CHECK-NEXT: ucvtf.4s v1, v1
; CHECK-NEXT: stp q1, q0, [x0]
; CHECK-NEXT: ret
@@ -68,14 +67,13 @@ define void @insert_vec_v23i32_uaddlv_from_v8i16(ptr %0) {
; CHECK-LABEL: insert_vec_v23i32_uaddlv_from_v8i16:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: movi.2d v0, #0000000000000000
-; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: str wzr, [x0, #88]
; CHECK-NEXT: str xzr, [x0, #80]
; CHECK-NEXT: uaddlv.8h s1, v0
; CHECK-NEXT: stp q0, q0, [x0, #16]
; CHECK-NEXT: stp q0, q0, [x0, #48]
-; CHECK-NEXT: mov.s v2[0], v1[0]
-; CHECK-NEXT: ucvtf.4s v1, v2
+; CHECK-NEXT: fmov s1, s1
+; CHECK-NEXT: ucvtf.4s v1, v1
; CHECK-NEXT: str q1, [x0]
; CHECK-NEXT: ret
@@ -254,10 +252,9 @@ define void @insert_vec_v16i64_uaddlv_from_v4i16(ptr %0) {
; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4h s1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.s v2[0], v1[0]
-; CHECK-NEXT: fmov x8, d2
-; CHECK-NEXT: mov.d x9, v2[1]
-; CHECK-NEXT: movi.2d v2, #0000000000000000
+; CHECK-NEXT: fmov s1, s1
+; CHECK-NEXT: fmov x8, d1
+; CHECK-NEXT: mov.d x9, v1[1]
; CHECK-NEXT: ucvtf s1, x8
; CHECK-NEXT: ucvtf s3, x9
; CHECK-NEXT: mov.s v2[0], v1[0]
@@ -346,8 +343,8 @@ define void @insert_vec_v8i32_uaddlv_from_v4i32(ptr %0) {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: movi.2d v0, #0000000000000000
; CHECK-NEXT: stp xzr, xzr, [x0, #16]
-; CHECK-NEXT: uaddlv.4s d1, v0
-; CHECK-NEXT: mov.s v0[0], v1[0]
+; CHECK-NEXT: uaddlv.4s d0, v0
+; CHECK-NEXT: fmov s0, s0
; CHECK-NEXT: ucvtf.4s v0, v0
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
@@ -365,11 +362,10 @@ define void @insert_vec_v16i32_uaddlv_from_v4i32(ptr %0) {
; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v4i32:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: movi.2d v0, #0000000000000000
-; CHECK-NEXT: movi.2d v2, #0000000000000000
; CHECK-NEXT: uaddlv.4s d1, v0
; CHECK-NEXT: stp q0, q0, [x0, #32]
-; CHECK-NEXT: mov.s v2[0], v1[0]
-; CHECK-NEXT: ucvtf.4s v1, v2
+; CHECK-NEXT: fmov s1, s1
+; CHECK-NEXT: ucvtf.4s v1, v1
; CHECK-NEXT: stp q1, q0, [x0]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/build-vector-extract.ll b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
index f3e40021b616a2..0cfbc939c12a82 100644
--- a/llvm/test/CodeGen/AArch64/build-vector-extract.ll
+++ b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
@@ -15,9 +15,7 @@ define <2 x i64> @extract0_i32_zext_insert0_i64_undef(<4 x i32> %x) {
define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
; CHECK-LABEL: extract0_i32_zext_insert0_i64_zero:
; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: fmov s0, s0
; CHECK-NEXT: ret
%e = extractelement <4 x i32> %x, i32 0
%z = zext i32 %e to i64
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 2483b476799f61..0fb3cb2643d651 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -321,9 +321,8 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
; CHECK-SD-NEXT: ldr d0, [x0]
; CHECK-SD-NEXT: ldr d1, [x1]
; CHECK-SD-NEXT: umull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT: fmov s1, s1
; CHECK-SD-NEXT: uaddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
@@ -406,10 +405,9 @@ define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v5i8_nomla:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr d0, [x0]
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT: fmov s1, s1
; CHECK-SD-NEXT: uaddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
@@ -447,9 +445,8 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
; CHECK-SD-NEXT: ldr d0, [x0]
; CHECK-SD-NEXT: ldr d1, [x1]
; CHECK-SD-NEXT: smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT: fmov s1, s1
; CHECK-SD-NEXT: saddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
@@ -533,14 +530,12 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: smull v2.8h, v2.8b, v3.8b
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT: sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT: sshll2 v5.4s, v2.8h, #0
-; CHECK-SD-NEXT: mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT: mov v1.s[0], v5.s[0]
-; CHECK-SD-NEXT: saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT: sshll2 v1.4s, v2.8h, #0
+; CHECK-SD-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-SD-NEXT: fmov s1, s1
+; CHECK-SD-NEXT: fmov s3, s3
; CHECK-SD-NEXT: saddw v1.4s, v1.4s, v2.4h
+; CHECK-SD-NEXT: saddw v0.4s, v3.4s, v0.4h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
@@ -682,14 +677,12 @@ define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: sshll v1.8h, v2.8b, #0
-; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT: sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT: sshll2 v5.4s, v1.8h, #0
-; CHECK-SD-NEXT: mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT: mov v2.s[0], v5.s[0]
-; CHECK-SD-NEXT: saddw v0.4s, v3.4s, v0.4h
-; CHECK-SD-NEXT: saddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT: sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT: fmov s2, s2
+; CHECK-SD-NEXT: fmov s3, s3
+; CHECK-SD-NEXT: saddw v0.4s, v2.4s, v0.4h
+; CHECK-SD-NEXT: saddw v1.4s, v3.4s, v1.4h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
@@ -2310,7 +2303,6 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-LABEL: test_udot_v25i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldp q3, q0, [x1]
-; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
; CHECK-SD-NEXT: ldp q2, q1, [x0]
; CHECK-SD-NEXT: umull2 v4.8h, v0.16b, v1.16b
; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
@@ -2319,10 +2311,10 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-NEXT: ushll v3.4s, v4.4h, #0
; CHECK-SD-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
; CHECK-SD-NEXT: uaddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT: fmov s3, s3
; CHECK-SD-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: uaddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT: uaddw v2.4s, v3.4s, v2.4h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
@@ -2637,20 +2629,19 @@ entry:
define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v25i8_nomla:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldp q2, q1, [x0]
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: ushll2 v3.8h, v1.16b, #0
-; CHECK-SD-NEXT: ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-SD-NEXT: uaddl v1.4s, v4.4h, v1.4h
-; CHECK-SD-NEXT: mov v0.s[0], v3.s[0]
-; CHECK-SD-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v2.4h
-; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: ldp q1, q0, [x0]
+; CHECK-SD-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT: uaddl2 v4.4s, v3.8h, v0.8h
+; CHECK-SD-NEXT: uaddl v0.4s, v3.4h, v0.4h
+; CHECK-SD-NEXT: fmov s2, s2
+; CHECK-SD-NEXT: uaddw2 v3.4s, v4.4s, v1.8h
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT: uaddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
@@ -2739,7 +2730,6 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-LABEL: test_sdot_v25i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldp q3, q0, [x1]
-; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
; CHECK-SD-NEXT: ldp q2, q1, [x0]
; CHECK-SD-NEXT: smull2 v4.8h, v0.16b, v1.16b
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
@@ -2748,10 +2738,10 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-NEXT: sshll v3.4s, v4.4h, #0
; CHECK-SD-NEXT: saddl2 v4.4s, v1.8h, v0.8h
; CHECK-SD-NEXT: saddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT: fmov s3, s3
; CHECK-SD-NEXT: saddw2 v1.4s, v4.4s, v2.8h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT: saddw v2.4s, v3.4s, v2.4h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
@@ -3073,203 +3063,201 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25
; CHECK-SD-NEXT: add x8, sp, #224
; CHECK-SD-NEXT: ldr b1, [sp, #16]
; CHECK-SD-NEXT: ldr b2, [sp, #280]
-; CHECK-SD-NEXT: add x9, sp, #240
+; CHECK-SD-NEXT: add x9, sp, #288
; CHECK-SD-NEXT: ldr b4, [sp, #80]
; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #24
-; CHECK-SD-NEXT: add x10, sp, #48
+; CHECK-SD-NEXT: ldr b5, [sp, #152]
; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #232
-; CHECK-SD-NEXT: add x11, sp, #96
-; CHECK-SD-NEXT: ldr b5, [sp, #152]
-; CHECK-SD-NEXT: add x12, sp, #168
-; CHECK-SD-NEXT: ldr b6, [sp, #616]
+; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #40
+; CHECK-SD-NEXT: add x10, sp, #296
+; CHECK-SD-NEXT: add x11, sp, #64
; CHECK-SD-NEXT: ld1 { v0.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #32
-; CHECK-SD-NEXT: fmov s3, w0
+; CHECK-SD-NEXT: ldr b6, [sp, #616]
; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #288
+; CHECK-SD-NEXT: add x8, sp, #240
+; CHECK-SD-NEXT: ld1 { v2.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #96
+; CHECK-SD-NEXT: fmov s3, w0
; CHECK-SD-NEXT: ldr b7, [sp, #416]
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #40
-; CHECK-SD-NEXT: ldr b22, [sp, #744]
-; CHECK-SD-NEXT: ld1 { v0.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #248
-; CHECK-SD-NEXT: mov v3.b[1], w1
-; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
; CHECK-SD-NEXT: add x8, sp, #88
-; CHECK-SD-NEXT: ldr b23, [sp, #544]
+; CHECK-SD-NEXT: ldr b16, [sp, #344]
+; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9]
; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #256
-; CHECK-SD-NEXT: ldr b19, [sp, #680]
-; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
-; CHECK-SD-NEXT: add x9, sp, #296
-; CHECK-SD-NEXT: ldr b20, [sp, #480]
-; CHECK-SD-NEXT: ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT: add x10, sp, #160
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT: add x11, sp, #304
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
-; CHECK-SD-NEXT: ld1 { v0.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #56
-; CHECK-SD-NEXT: add x10, sp, #264
-; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #64
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x11]
+; CHECK-SD-NEXT: add x8, sp, #248
+; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: mov v3.b[1], w1
+; CHECK-SD-NEXT: ldr b17, [sp, #144]
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #48
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #256
+; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8]
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #56
+; CHECK-SD-NEXT: add x8, sp, #264
+; CHECK-SD-NEXT: ldr b20, [sp, #744]
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x9]
; CHECK-SD-NEXT: add x9, sp, #272
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x12]
-; CHECK-SD-NEXT: add x11, sp, #72
-; CHECK-SD-NEXT: ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #312
+; CHECK-SD-NEXT: ldr b21, [sp, #544]
+; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #304
; CHECK-SD-NEXT: mov v3.b[2], w2
-; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v2.b }[3], [x10]
+; CHECK-SD-NEXT: add x10, sp, #72
+; CHECK-SD-NEXT: smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #168
+; CHECK-SD-NEXT: ldr b17, [sp, #680]
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #104
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT: ld1 { v1.b }[6], [x11]
+; CHECK-SD-NEXT: add x11, sp, #312
; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #112
-; CHECK-SD-NEXT: add x10, sp, #128
+; CHECK-SD-NEXT: add x8, sp, #320
+; CHECK-SD-NEXT: ld1 { v2.b }[4], [x11]
; CHECK-SD-NEXT: ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #320
-; CHECK-SD-NEXT: ldr b21, [sp, #552]
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
-; CHECK-SD-NEXT: add x9, sp, #176
-; CHECK-SD-NEXT: ld1 { v1.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #624
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #120
-; CHECK-SD-NEXT: add x9, sp, #328
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9]
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: add x11, sp, #192
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #632
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x9, sp, #640
+; CHECK-SD-NEXT: add x9, sp, #624
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT: add x10, sp, #112
+; CHECK-SD-NEXT: add x11, sp, #176
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x10]
+; CHECK-SD-NEXT: add x9, sp, #632
+; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x11]
+; CHECK-SD-NEXT: add x8, sp, #328
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT: add x10, sp, #120
+; CHECK-SD-NEXT: add x11, sp, #184
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #640
+; CHECK-SD-NEXT: add x9, sp, #128
+; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11]
; CHECK-SD-NEXT: add x8, sp, #336
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x10]
+; CHECK-SD-NEXT: add x11, sp, #192
+; CHECK-SD-NEXT: ldr b18, [sp, #480]
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT: add x9, sp, #200
+; CHECK-SD-NEXT: ldr b19, [sp, #552]
; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #656
-; CHECK-SD-NEXT: smull v23.8h, v23.8b, v22.8b
+; CHECK-SD-NEXT: add x8, sp, #648
; CHECK-SD-NEXT: ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #648
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #200
-; CHECK-SD-NEXT: add x10, sp, #136
-; CHECK-SD-NEXT: ldr b22, [sp, #352]
-; CHECK-SD-NEXT: add x12, sp, #360
-; CHECK-SD-NEXT: mov v3.b[3], w3
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #424
+; CHECK-SD-NEXT: smull v21.8h, v21.8b, v20.8b
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #656
+; CHECK-SD-NEXT: ldr b20, [sp, #352]
; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT: add x9, sp, #208
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x11]
-; CHECK-SD-NEXT: add x11, sp, #424
+; CHECK-SD-NEXT: add x9, sp, #136
; CHECK-SD-NEXT: add x10, sp, #488
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #560
-; CHECK-SD-NEXT: ld1 { v20.b }[1], [x10]
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #440
-; CHECK-SD-NEXT: ld1 { v21.b }[1], [x11]
; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
; CHECK-SD-NEXT: add x8, sp, #432
-; CHECK-SD-NEXT: ld1 { v22.b }[1], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT: add x9, sp, #208
+; CHECK-SD-NEXT: add x11, sp, #560
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #440
+; CHECK-SD-NEXT: add x12, sp, #360
+; CHECK-SD-NEXT: ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v19.b }[1], [x11]
+; CHECK-SD-NEXT: ld1 { v20.b }[1], [x12]
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #688
+; CHECK-SD-NEXT: mov v3.b[3], w3
+; CHECK-SD-NEXT: ld1 { v17.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #696
; CHECK-SD-NEXT: add x11, sp, #496
; CHECK-SD-NEXT: add x12, sp, #568
; CHECK-SD-NEXT: add x13, sp, #368
-; CHECK-SD-NEXT: ld1 { v20.b }[2], [x11]
-; CHECK-SD-NEXT: ld1 { v21.b }[2], [x12]
-; CHECK-SD-NEXT: ld1 { v22.b }[2], [x13]
+; CHECK-SD-NEXT: ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT: ld1 { v19.b }[2], [x12]
+; CHECK-SD-NEXT: ld1 { v20.b }[2], [x13]
; CHECK-SD-NEXT: add x10, sp, #448
+; CHECK-SD-NEXT: ld1 { v17.b }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #704
; CHECK-SD-NEXT: mov v3.b[4], w4
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #688
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #504
; CHECK-SD-NEXT: add x11, sp, #576
-; CHECK-SD-NEXT: ld1 { v19.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #696
; CHECK-SD-NEXT: add x12, sp, #376
-; CHECK-SD-NEXT: ld1 { v21.b }[3], [x11]
-; CHECK-SD-NEXT: ld1 { v22.b }[3], [x12]
+; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10]
+; CHECK-SD-NEXT: ld1 { v19.b }[3], [x11]
+; CHECK-SD-NEXT: ld1 { v17.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v20.b }[3], [x12]
+; CHECK-SD-NEXT: add x9, sp, #712
; CHECK-SD-NEXT: add x11, sp, #512
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #504
; CHECK-SD-NEXT: add x12, sp, #584
-; CHECK-SD-NEXT: ld1 { v19.b }[2], [x9]
-; CHECK-SD-NEXT: add x9, sp, #704
-; CHECK-SD-NEXT: ld1 { v20.b }[3], [x10]
; CHECK-SD-NEXT: add x13, sp, #384
; CHECK-SD-NEXT: mov v3.b[5], w5
-; CHECK-SD-NEXT: ld1 { v21.b }[4], [x12]
-; CHECK-SD-NEXT: ld1 { v22.b }[4], [x13]
+; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT: ld1 { v19.b }[4], [x12]
+; CHECK-SD-NEXT: ld1 { v17.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v20.b }[4], [x13]
; CHECK-SD-NEXT: add x10, sp, #456
-; CHECK-SD-NEXT: ldr b16, [sp, #344]
-; CHECK-SD-NEXT: ld1 { v19.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #712
-; CHECK-SD-NEXT: ld1 { v20.b }[4], [x11]
-; CHECK-SD-NEXT: ldr b17, [sp, #144]
+; CHECK-SD-NEXT: add x9, sp, #720
; CHECK-SD-NEXT: ld1 { v7.b }[5], [x10]
; CHECK-SD-NEXT: add x10, sp, #520
; CHECK-SD-NEXT: add x11, sp, #592
; CHECK-SD-NEXT: add x12, sp, #392
+; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v17.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v19.b }[5], [x11]
+; CHECK-SD-NEXT: ld1 { v20.b }[5], [x12]
; CHECK-SD-NEXT: mov v3.b[6], w6
-; CHECK-SD-NEXT: ld1 { v19.b }[4], [x9]
-; CHECK-SD-NEXT: add x9, sp, #720
-; CHECK-SD-NEXT: ld1 { v20.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v21.b }[5], [x11]
-; CHECK-SD-NEXT: ld1 { v22.b }[5], [x12]
-; CHECK-SD-NEXT: smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT: add x9, sp, #728
; CHECK-SD-NEXT: add x8, sp, #664
; CHECK-SD-NEXT: add x10, sp, #464
; CHECK-SD-NEXT: add x11, sp, #528
-; CHECK-SD-NEXT: ld1 { v19.b }[5], [x9]
-; CHECK-SD-NEXT: add x9, sp, #728
; CHECK-SD-NEXT: add x12, sp, #600
+; CHECK-SD-NEXT: ld1 { v17.b }[6], [x9]
; CHECK-SD-NEXT: add x13, sp, #400
; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v20.b }[6], [x11]
-; CHECK-SD-NEXT: ld1 { v21.b }[6], [x12]
-; CHECK-SD-NEXT: ld1 { v22.b }[6], [x13]
+; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT: ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT: ld1 { v20.b }[6], [x13]
; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[6], [x9]
; CHECK-SD-NEXT: add x9, sp, #736
; CHECK-SD-NEXT: mov v3.b[7], w7
-; CHECK-SD-NEXT: sshll v18.4s, v16.4h, #0
-; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
; CHECK-SD-NEXT: add x8, sp, #672
; CHECK-SD-NEXT: add x10, sp, #472
-; CHECK-SD-NEXT: add x11, sp, #608
-; CHECK-SD-NEXT: ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v17.b }[7], [x9]
; CHECK-SD-NEXT: add x9, sp, #536
+; CHECK-SD-NEXT: add x11, sp, #608
; CHECK-SD-NEXT: add x12, sp, #408
-; CHECK-SD-NEXT: ld1 { v20.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v21.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v22.b }[7], [x12]
+; CHECK-SD-NEXT: sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT: ld1 { v18.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v19.b }[7], [x11]
+; CHECK-SD-NEXT: ld1 { v20.b }[7], [x12]
; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
-; CHECK-SD-NEXT: sshll v23.4s, v23.4h, #0
+; CHECK-SD-NEXT: sshll v21.4s, v21.4h, #0
; CHECK-SD-NEXT: smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT: smull v1.8h, v4.8b, v2.8b
; CHECK-SD-NEXT: smull v2.8h, v3.8b, v5.8b
-; CHECK-SD-NEXT: smull v3.8h, v20.8b, v19.8b
-; CHECK-SD-NEXT: smull v4.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT: mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT: smull v5.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT: mov v16.s[0], v23.s[0]
-; CHECK-SD-NEXT: saddl2 v6.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT: smull v3.8h, v18.8b, v17.8b
+; CHECK-SD-NEXT: fmov s5, s16
+; CHECK-SD-NEXT: smull v4.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT: smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT: fmov s7, s21
+; CHECK-SD-NEXT: saddl2 v16.4s, v2.8h, v1.8h
; CHECK-SD-NEXT: saddl v1.4s, v2.4h, v1.4h
; CHECK-SD-NEXT: saddl2 v2.4s, v4.8h, v3.8h
; CHECK-SD-NEXT: saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT: saddw v4.4s, v17.4s, v0.4h
-; CHECK-SD-NEXT: saddw v7.4s, v16.4s, v5.4h
-; CHECK-SD-NEXT: saddw2 v0.4s, v6.4s, v0.8h
+; CHECK-SD-NEXT: saddw v4.4s, v5.4s, v0.4h
+; CHECK-SD-NEXT: saddw v5.4s, v7.4s, v6.4h
+; CHECK-SD-NEXT: saddw2 v0.4s, v16.4s, v0.8h
; CHECK-SD-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT: saddw2 v2.4s, v2.4s, v5.8h
-; CHECK-SD-NEXT: add v3.4s, v3.4s, v7.4s
+; CHECK-SD-NEXT: saddw2 v2.4s, v2.4s, v6.8h
+; CHECK-SD-NEXT: add v3.4s, v3.4s, v5.4s
; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT: add v1.4s, v3.4s, v2.4s
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
@@ -3827,117 +3815,115 @@ define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %
; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: .cfi_offset w29, -16
-; CHECK-SD-NEXT: fmov s0, w0
+; CHECK-SD-NEXT: ldr b0, [sp, #16]
+; CHECK-SD-NEXT: add x8, sp, #24
; CHECK-SD-NEXT: ldr b1, [sp, #80]
-; CHECK-SD-NEXT: add x10, sp, #88
-; CHECK-SD-NEXT: ldr b2, [sp, #16]
-; CHECK-SD-NEXT: add x9, sp, #96
+; CHECK-SD-NEXT: fmov s2, w0
+; CHECK-SD-NEXT: add x11, sp, #104
+; CHECK-SD-NEXT: add x12, sp, #48
+; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #88
; CHECK-SD-NEXT: ldr b3, [sp, #480]
-; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #24
-; CHECK-SD-NEXT: ldr b4, [sp, #352]
-; CHECK-SD-NEXT: mov v0.b[1], w1
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x10]
-; CHECK-SD-NEXT: add x11, sp, #488
-; CHECK-SD-NEXT: add x10, sp, #360
-; CHECK-SD-NEXT: ldr b5, [sp, #416]
-; CHECK-SD-NEXT: add x8, sp, #104
-; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]
-; CHECK-SD-NEXT: add x9, sp, #32
-; CHECK-SD-NEXT: ld1 { v3.b }[1], [x11]
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT: add x11, sp, #424
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x10]
-; CHECK-SD-NEXT: mov v0.b[2], w2
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT: add x9, sp, #368
-; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #32
+; CHECK-SD-NEXT: ldr b5, [sp, #352]
+; CHECK-SD-NEXT: mov v2.b[1], w1
+; CHECK-SD-NEXT: ldr b4, [sp, #416]
+; CHECK-SD-NEXT: add x10, sp, #56
+; CHECK-SD-NEXT: ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #96
+; CHECK-SD-NEXT: add x13, sp, #128
+; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #40
-; CHECK-SD-NEXT: add x12, sp, #496
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x9]
-; CHECK-SD-NEXT: add x8, sp, #432
-; CHECK-SD-NEXT: ld1 { v3.b }[2], [x12]
-; CHECK-SD-NEXT: add x13, sp, #48
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT: mov v0.b[3], w3
-; CHECK-SD-NEXT: add x10, sp, #112
-; CHECK-SD-NEXT: add x8, sp, #504
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x13]
-; CHECK-SD-NEXT: add x13, sp, #376
-; CHECK-SD-NEXT: ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT: add x13, sp, #440
-; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT: add x11, sp, #120
-; CHECK-SD-NEXT: add x8, sp, #56
-; CHECK-SD-NEXT: mov v0.b[4], w4
-; CHECK-SD-NEXT: add x13, sp, #512
-; CHECK-SD-NEXT: ld1 { v1.b }[5], [x11]
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #384
-; CHECK-SD-NEXT: add x11, sp, #448
-; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13]
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT: add x12, sp, #128
-; CHECK-SD-NEXT: add x10, sp, #64
-; CHECK-SD-NEXT: add x8, sp, #520
-; CHECK-SD-NEXT: mov v0.b[5], w5
-; CHECK-SD-NEXT: ld1 { v1.b }[6], [x12]
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #392
-; CHECK-SD-NEXT: add x11, sp, #456
+; CHECK-SD-NEXT: add x9, sp, #64
; CHECK-SD-NEXT: ldr b6, [sp, #144]
; CHECK-SD-NEXT: ldr b7, [sp, #544]
-; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
+; CHECK-SD-NEXT: mov v2.b[2], w2
+; CHECK-SD-NEXT: add x8, sp, #72
+; CHECK-SD-NEXT: ld1 { v1.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #112
; CHECK-SD-NEXT: sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT: mov v0.b[6], w6
-; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9]
-; CHECK-SD-NEXT: add x8, sp, #528
-; CHECK-SD-NEXT: add x9, sp, #400
-; CHECK-SD-NEXT: add x10, sp, #464
; CHECK-SD-NEXT: sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT: ld1 { v3.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x10]
-; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT: add x14, sp, #72
-; CHECK-SD-NEXT: mov v0.b[7], w7
-; CHECK-SD-NEXT: sshll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT: add x8, sp, #536
-; CHECK-SD-NEXT: add x9, sp, #408
-; CHECK-SD-NEXT: add x10, sp, #472
-; CHECK-SD-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT: ld1 { v2.b }[7], [x14]
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x10]
-; CHECK-SD-NEXT: mov v16.s[0], v6.s[0]
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x12]
+; CHECK-SD-NEXT: add x12, sp, #488
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x12]
+; CHECK-SD-NEXT: add x12, sp, #360
+; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11]
+; CHECK-SD-NEXT: add x11, sp, #424
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x12]
+; CHECK-SD-NEXT: mov v2.b[3], w3
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT: add x11, sp, #496
+; CHECK-SD-NEXT: add x12, sp, #120
+; CHECK-SD-NEXT: ld1 { v3.b }[2], [x11]
+; CHECK-SD-NEXT: add x11, sp, #368
+; CHECK-SD-NEXT: ld1 { v1.b }[5], [x12]
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x11]
+; CHECK-SD-NEXT: add x11, sp, #432
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT: mov v2.b[4], w4
+; CHECK-SD-NEXT: add x11, sp, #440
+; CHECK-SD-NEXT: add x10, sp, #504
+; CHECK-SD-NEXT: add x12, sp, #376
+; CHECK-SD-NEXT: ld1 { v1.b }[6], [x13]
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x12]
+; CHECK-SD-NEXT: add x12, sp, #512
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #448
+; CHECK-SD-NEXT: add x13, sp, #384
+; CHECK-SD-NEXT: mov v2.b[5], w5
+; CHECK-SD-NEXT: add x10, sp, #136
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT: ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x11]
+; CHECK-SD-NEXT: add x9, sp, #520
+; CHECK-SD-NEXT: add x10, sp, #392
+; CHECK-SD-NEXT: add x11, sp, #456
+; CHECK-SD-NEXT: sshll v6.4s, v6.4h, #0
+; CHECK-SD-NEXT: sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x10]
+; CHECK-SD-NEXT: mov v2.b[6], w6
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x11]
+; CHECK-SD-NEXT: add x9, sp, #464
+; CHECK-SD-NEXT: add x10, sp, #528
+; CHECK-SD-NEXT: add x11, sp, #400
+; CHECK-SD-NEXT: ld1 { v0.b }[7], [x8]
+; CHECK-SD-NEXT: fmov s6, s6
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT: add x10, sp, #536
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT: mov v2.b[7], w7
+; CHECK-SD-NEXT: add x9, sp, #472
+; CHECK-SD-NEXT: add x11, sp, #408
+; CHECK-SD-NEXT: fmov s7, s7
; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT: mov v17.s[0], v7.s[0]
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x11]
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
; CHECK-SD-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-SD-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-SD-NEXT: sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT: saddl v7.4s, v0.4h, v1.4h
-; CHECK-SD-NEXT: saddl2 v0.4s, v0.8h, v1.8h
-; CHECK-SD-NEXT: saddw v6.4s, v16.4s, v2.4h
-; CHECK-SD-NEXT: saddl v1.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT: saddl2 v3.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT: saddw v4.4s, v17.4s, v5.4h
-; CHECK-SD-NEXT: saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-SD-NEXT: add v6.4s, v7.4s, v6.4s
-; CHECK-SD-NEXT: saddw2 v2.4s, v3.4s, v5.8h
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT: add v0.4s, v6.4s, v0.4s
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT: sshll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT: saddl2 v16.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT: saddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT: saddl2 v2.4s, v5.8h, v3.8h
+; CHECK-SD-NEXT: saddl v3.4s, v5.4h, v3.4h
+; CHECK-SD-NEXT: saddw v5.4s, v6.4s, v0.4h
+; CHECK-SD-NEXT: saddw v6.4s, v7.4s, v4.4h
+; CHECK-SD-NEXT: saddw2 v0.4s, v16.4s, v0.8h
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT: saddw2 v2.4s, v2.4s, v4.8h
+; CHECK-SD-NEXT: add v3.4s, v3.4s, v6.4s
+; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: add v1.4s, v3.4s, v2.4s
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
@@ -4457,7 +4443,6 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr b0, [x0, #32]
; CHECK-SD-NEXT: ldr b1, [x1, #32]
-; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
; CHECK-SD-NEXT: ldp q3, q4, [x1]
; CHECK-SD-NEXT: umull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT: ldp q1, q2, [x0]
@@ -4466,15 +4451,15 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-NEXT: umull2 v2.8h, v4.16b, v2.16b
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: umull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT: mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT: uaddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT: uaddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT: fmov s0, s0
+; CHECK-SD-NEXT: uaddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT: uaddl2 v3.4s, v1.8h, v2.8h
; CHECK-SD-NEXT: uaddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT: uaddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT: uaddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT: add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
; CHECK-SD-NEXT: add w0, w8, w2
@@ -4903,23 +4888,22 @@ entry:
define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
; CHECK-SD-LABEL: test_udot_v33i8_nomla:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ldr b1, [x0, #32]
-; CHECK-SD-NEXT: ldp q3, q2, [x0]
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT: ldr b0, [x0, #32]
+; CHECK-SD-NEXT: ldp q2, q1, [x0]
+; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: ushll v3.8h, v1.8b, #0
; CHECK-SD-NEXT: ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT: ushll v5.8h, v3.8b, #0
+; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-SD-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT: ushll2 v3.8h, v3.16b, #0
-; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT: uaddl2 v6.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT: mov v0.s[0], v1.s[0]
-; CHECK-SD-NEXT: uaddl2 v1.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT: uaddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT: add v1.4s, v6.4s, v1.4s
-; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v5.4h
-; CHECK-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: uaddl2 v6.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT: uaddl2 v5.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT: uaddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT: fmov s0, s0
+; CHECK-SD-NEXT: add v5.4s, v6.4s, v5.4s
; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v4.4h
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT: uaddw v0.4s, v0.4s, v3.4h
; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
@@ -5054,7 +5038,6 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr b0, [x0, #32]
; CHECK-SD-NEXT: ldr b1, [x1, #32]
-; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
; CHECK-SD-NEXT: ldp q3, q4, [x1]
; CHECK-SD-NEXT: smull v0.8h, v1.8b, v0.8b
; CHECK-SD-NEXT: ldp q1, q2, [x0]
@@ -5063,15 +5046,15 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
; CHECK-SD-NEXT: smull2 v2.8h, v4.16b, v2.16b
; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: smull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT: mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT: saddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT: saddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT: fmov s0, s0
+; CHECK-SD-NEXT: saddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT: saddl2 v3.4s, v1.8h, v2.8h
; CHECK-SD-NEXT: saddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT: saddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT: saddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT: add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: saddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT: saddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s0
; CHECK-SD-NEXT: add w0, w8, w2
@@ -5506,268 +5489,266 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
; CHECK-SD-NEXT: ldr b0, [sp, #344]
; CHECK-SD-NEXT: add x8, sp, #352
; CHECK-SD-NEXT: ldr b1, [sp, #80]
-; CHECK-SD-NEXT: ldr b2, [sp, #216]
-; CHECK-SD-NEXT: add x9, sp, #96
-; CHECK-SD-NEXT: add x10, sp, #104
-; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #88
+; CHECK-SD-NEXT: add x9, sp, #88
+; CHECK-SD-NEXT: ldr b3, [sp, #216]
; CHECK-SD-NEXT: ldr b4, [sp, #408]
-; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[1], [x9]
; CHECK-SD-NEXT: add x8, sp, #360
-; CHECK-SD-NEXT: add x12, sp, #248
-; CHECK-SD-NEXT: add x13, sp, #432
-; CHECK-SD-NEXT: add x11, sp, #384
-; CHECK-SD-NEXT: ldr b5, [sp, #144]
+; CHECK-SD-NEXT: add x9, sp, #224
+; CHECK-SD-NEXT: add x12, sp, #416
+; CHECK-SD-NEXT: add x10, sp, #376
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #96
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x12]
; CHECK-SD-NEXT: ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #224
-; CHECK-SD-NEXT: ldr b6, [sp, #280]
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]
; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]
; CHECK-SD-NEXT: add x8, sp, #368
; CHECK-SD-NEXT: add x9, sp, #232
-; CHECK-SD-NEXT: ldr b16, [sp, #744]
-; CHECK-SD-NEXT: ldr b17, [sp, #480]
+; CHECK-SD-NEXT: add x12, sp, #112
+; CHECK-SD-NEXT: ldr b6, [sp, #744]
+; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #104
+; CHECK-SD-NEXT: add x11, sp, #384
; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #376
+; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9]
+; CHECK-SD-NEXT: add x8, sp, #392
+; CHECK-SD-NEXT: add x9, sp, #400
+; CHECK-SD-NEXT: ldr b5, [sp, #144]
+; CHECK-SD-NEXT: ldr b7, [sp, #480]
; CHECK-SD-NEXT: ldr b18, [sp, #936]
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v1.b }[3], [x10]
-; CHECK-SD-NEXT: add x9, sp, #240
-; CHECK-SD-NEXT: add x10, sp, #392
; CHECK-SD-NEXT: ldr b19, [sp, #672]
-; CHECK-SD-NEXT: ldr b7, [sp, #16]
-; CHECK-SD-NEXT: ld1 { v0.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #112
-; CHECK-SD-NEXT: ldr b21, [sp, #1000]
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #416
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #120
-; CHECK-SD-NEXT: add x9, sp, #400
-; CHECK-SD-NEXT: ld1 { v0.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #128
-; CHECK-SD-NEXT: ldr b22, [sp, #736]
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT: ldr b20, [sp, #808]
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #240
+; CHECK-SD-NEXT: ld1 { v1.b }[4], [x12]
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10]
; CHECK-SD-NEXT: add x12, sp, #424
-; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT: add x10, sp, #120
; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12]
-; CHECK-SD-NEXT: add x12, sp, #152
-; CHECK-SD-NEXT: add x8, sp, #136
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x12]
-; CHECK-SD-NEXT: add x12, sp, #440
-; CHECK-SD-NEXT: ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v1.b }[6], [x11]
-; CHECK-SD-NEXT: add x11, sp, #288
-; CHECK-SD-NEXT: add x10, sp, #256
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #296
+; CHECK-SD-NEXT: add x12, sp, #752
+; CHECK-SD-NEXT: ldr b22, [sp, #544]
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #248
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x11]
+; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #432
+; CHECK-SD-NEXT: add x12, sp, #760
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x10]
+; CHECK-SD-NEXT: add x11, sp, #256
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #128
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x11]
+; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #440
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #768
+; CHECK-SD-NEXT: add x10, sp, #264
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
; CHECK-SD-NEXT: ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #160
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x9]
-; CHECK-SD-NEXT: add x10, sp, #168
-; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x12]
-; CHECK-SD-NEXT: add x12, sp, #448
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x11]
-; CHECK-SD-NEXT: add x11, sp, #304
-; CHECK-SD-NEXT: add x8, sp, #464
-; CHECK-SD-NEXT: add x13, sp, #768
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x10]
-; CHECK-SD-NEXT: add x10, sp, #176
-; CHECK-SD-NEXT: add x9, sp, #264
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x12]
-; CHECK-SD-NEXT: add x12, sp, #456
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #760
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #448
+; CHECK-SD-NEXT: add x8, sp, #152
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #776
+; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9]
; CHECK-SD-NEXT: add x9, sp, #272
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #312
-; CHECK-SD-NEXT: fmov s3, w0
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x12]
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
; CHECK-SD-NEXT: ld1 { v6.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #320
-; CHECK-SD-NEXT: add x12, sp, #680
-; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: ld1 { v19.b }[1], [x12]
-; CHECK-SD-NEXT: add x12, sp, #776
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #752
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x8]
-; CHECK-SD-NEXT: add x10, sp, #24
-; CHECK-SD-NEXT: smull v22.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #496
-; CHECK-SD-NEXT: mov v3.b[1], w1
-; CHECK-SD-NEXT: add x9, sp, #192
-; CHECK-SD-NEXT: ldr b20, [sp, #472]
-; CHECK-SD-NEXT: ldr b23, [sp, #208]
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
-; CHECK-SD-NEXT: add x11, sp, #488
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v17.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #944
-; CHECK-SD-NEXT: add x9, sp, #328
-; CHECK-SD-NEXT: ld1 { v18.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #688
+; CHECK-SD-NEXT: add x8, sp, #488
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #456
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT: add x8, sp, #160
+; CHECK-SD-NEXT: add x9, sp, #784
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #496
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #464
+; CHECK-SD-NEXT: add x10, sp, #944
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #792
+; CHECK-SD-NEXT: ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT: add x10, sp, #552
+; CHECK-SD-NEXT: add x11, sp, #952
; CHECK-SD-NEXT: ld1 { v6.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x13]
-; CHECK-SD-NEXT: ld1 { v19.b }[2], [x11]
-; CHECK-SD-NEXT: add x11, sp, #504
-; CHECK-SD-NEXT: ld1 { v17.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #952
-; CHECK-SD-NEXT: add x13, sp, #784
-; CHECK-SD-NEXT: ld1 { v18.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #32
-; CHECK-SD-NEXT: add x9, sp, #40
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x12]
-; CHECK-SD-NEXT: add x12, sp, #696
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #960
-; CHECK-SD-NEXT: ld1 { v19.b }[3], [x12]
-; CHECK-SD-NEXT: ld1 { v18.b }[3], [x11]
-; CHECK-SD-NEXT: add x10, sp, #512
-; CHECK-SD-NEXT: add x11, sp, #704
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x13]
-; CHECK-SD-NEXT: add x12, sp, #792
-; CHECK-SD-NEXT: sshll v24.4s, v22.4h, #0
-; CHECK-SD-NEXT: ld1 { v17.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #968
-; CHECK-SD-NEXT: ld1 { v19.b }[4], [x11]
-; CHECK-SD-NEXT: ld1 { v18.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #520
-; CHECK-SD-NEXT: add x11, sp, #976
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT: add x12, sp, #712
-; CHECK-SD-NEXT: smull v20.8h, v23.8b, v20.8b
-; CHECK-SD-NEXT: ld1 { v17.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[5], [x12]
-; CHECK-SD-NEXT: add x12, sp, #720
-; CHECK-SD-NEXT: ld1 { v18.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #528
-; CHECK-SD-NEXT: add x10, sp, #800
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT: add x10, sp, #536
-; CHECK-SD-NEXT: ldr b22, [sp, #872]
-; CHECK-SD-NEXT: ld1 { v17.b }[6], [x11]
+; CHECK-SD-NEXT: add x9, sp, #504
+; CHECK-SD-NEXT: ld1 { v22.b }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #800
+; CHECK-SD-NEXT: add x10, sp, #688
+; CHECK-SD-NEXT: ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT: add x11, sp, #560
+; CHECK-SD-NEXT: add x12, sp, #960
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #512
+; CHECK-SD-NEXT: ld1 { v22.b }[2], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x9]
+; CHECK-SD-NEXT: add x9, sp, #680
+; CHECK-SD-NEXT: add x11, sp, #968
+; CHECK-SD-NEXT: ld1 { v19.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #816
+; CHECK-SD-NEXT: ld1 { v18.b }[3], [x12]
+; CHECK-SD-NEXT: ld1 { v20.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #520
+; CHECK-SD-NEXT: add x12, sp, #976
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #696
+; CHECK-SD-NEXT: fmov s2, w0
+; CHECK-SD-NEXT: ld1 { v19.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #824
+; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT: ld1 { v20.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #568
+; CHECK-SD-NEXT: add x11, sp, #576
+; CHECK-SD-NEXT: ld1 { v22.b }[3], [x10]
+; CHECK-SD-NEXT: add x10, sp, #704
+; CHECK-SD-NEXT: mov v2.b[1], w1
+; CHECK-SD-NEXT: ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #832
+; CHECK-SD-NEXT: ld1 { v18.b }[5], [x12]
+; CHECK-SD-NEXT: ld1 { v20.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #528
+; CHECK-SD-NEXT: add x12, sp, #992
+; CHECK-SD-NEXT: ld1 { v22.b }[4], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x9]
+; CHECK-SD-NEXT: add x9, sp, #712
+; CHECK-SD-NEXT: ld1 { v19.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #840
; CHECK-SD-NEXT: add x11, sp, #984
-; CHECK-SD-NEXT: ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT: ld1 { v20.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #584
; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT: add x11, sp, #992
-; CHECK-SD-NEXT: add x12, sp, #728
-; CHECK-SD-NEXT: ldr b23, [sp, #608]
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #880
-; CHECK-SD-NEXT: ld1 { v17.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[7], [x12]
-; CHECK-SD-NEXT: add x10, sp, #816
-; CHECK-SD-NEXT: ld1 { v18.b }[7], [x11]
-; CHECK-SD-NEXT: add x11, sp, #552
-; CHECK-SD-NEXT: add x12, sp, #616
-; CHECK-SD-NEXT: mov v3.b[2], w2
-; CHECK-SD-NEXT: ld1 { v22.b }[1], [x9]
-; CHECK-SD-NEXT: ld1 { v23.b }[1], [x12]
+; CHECK-SD-NEXT: ld1 { v22.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #720
+; CHECK-SD-NEXT: add x11, sp, #592
+; CHECK-SD-NEXT: ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #848
+; CHECK-SD-NEXT: ldr b16, [sp, #472]
+; CHECK-SD-NEXT: ld1 { v20.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #536
+; CHECK-SD-NEXT: ld1 { v18.b }[7], [x12]
+; CHECK-SD-NEXT: ld1 { v22.b }[6], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #728
+; CHECK-SD-NEXT: ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #856
+; CHECK-SD-NEXT: ldr b17, [sp, #208]
+; CHECK-SD-NEXT: ld1 { v20.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #600
+; CHECK-SD-NEXT: add x8, sp, #168
+; CHECK-SD-NEXT: ld1 { v22.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
; CHECK-SD-NEXT: smull v16.8h, v17.8b, v16.8b
-; CHECK-SD-NEXT: add x12, sp, #560
-; CHECK-SD-NEXT: add x9, sp, #888
-; CHECK-SD-NEXT: smull v17.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT: ldr b18, [sp, #808]
-; CHECK-SD-NEXT: ldr b19, [sp, #544]
-; CHECK-SD-NEXT: add x13, sp, #624
-; CHECK-SD-NEXT: ld1 { v22.b }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #864
+; CHECK-SD-NEXT: smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT: ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT: ldr b23, [sp, #736]
+; CHECK-SD-NEXT: add x10, sp, #880
+; CHECK-SD-NEXT: add x11, sp, #616
+; CHECK-SD-NEXT: mov v2.b[2], w2
+; CHECK-SD-NEXT: add x8, sp, #176
+; CHECK-SD-NEXT: smull v7.8h, v19.8b, v18.8b
+; CHECK-SD-NEXT: ldr b18, [sp, #1000]
+; CHECK-SD-NEXT: ldr b19, [sp, #872]
+; CHECK-SD-NEXT: smull v17.8h, v22.8b, v20.8b
+; CHECK-SD-NEXT: ldr b20, [sp, #608]
+; CHECK-SD-NEXT: ldr b21, [sp, #280]
+; CHECK-SD-NEXT: smull v22.8h, v23.8b, v18.8b
+; CHECK-SD-NEXT: ldr b18, [sp, #16]
+; CHECK-SD-NEXT: ld1 { v19.b }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v20.b }[1], [x11]
+; CHECK-SD-NEXT: add x9, sp, #288
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
+; CHECK-SD-NEXT: ld1 { v21.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #624
+; CHECK-SD-NEXT: ld1 { v18.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #888
+; CHECK-SD-NEXT: mov v2.b[3], w3
+; CHECK-SD-NEXT: ld1 { v19.b }[2], [x8]
+; CHECK-SD-NEXT: ld1 { v20.b }[2], [x9]
+; CHECK-SD-NEXT: add x10, sp, #296
+; CHECK-SD-NEXT: add x9, sp, #32
+; CHECK-SD-NEXT: ld1 { v21.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #632
+; CHECK-SD-NEXT: ld1 { v18.b }[2], [x9]
; CHECK-SD-NEXT: add x9, sp, #896
-; CHECK-SD-NEXT: ld1 { v18.b }[1], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #824
-; CHECK-SD-NEXT: add x10, sp, #48
-; CHECK-SD-NEXT: ld1 { v23.b }[2], [x13]
-; CHECK-SD-NEXT: mov v3.b[3], w3
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #832
-; CHECK-SD-NEXT: ld1 { v22.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v18.b }[2], [x11]
-; CHECK-SD-NEXT: ld1 { v19.b }[2], [x12]
-; CHECK-SD-NEXT: add x11, sp, #568
-; CHECK-SD-NEXT: add x12, sp, #632
-; CHECK-SD-NEXT: add x9, sp, #904
-; CHECK-SD-NEXT: add x13, sp, #640
-; CHECK-SD-NEXT: ld1 { v23.b }[3], [x12]
-; CHECK-SD-NEXT: add x12, sp, #576
-; CHECK-SD-NEXT: mov v3.b[4], w4
-; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #840
-; CHECK-SD-NEXT: add x10, sp, #56
-; CHECK-SD-NEXT: ld1 { v22.b }[4], [x9]
+; CHECK-SD-NEXT: add x8, sp, #184
+; CHECK-SD-NEXT: ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v20.b }[3], [x10]
+; CHECK-SD-NEXT: add x11, sp, #304
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT: add x8, sp, #40
+; CHECK-SD-NEXT: mov v2.b[4], w4
+; CHECK-SD-NEXT: ld1 { v21.b }[3], [x11]
+; CHECK-SD-NEXT: ld1 { v18.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #904
+; CHECK-SD-NEXT: add x9, sp, #640
+; CHECK-SD-NEXT: ld1 { v19.b }[4], [x8]
+; CHECK-SD-NEXT: add x10, sp, #312
+; CHECK-SD-NEXT: ld1 { v20.b }[4], [x9]
+; CHECK-SD-NEXT: add x9, sp, #48
+; CHECK-SD-NEXT: add x11, sp, #320
+; CHECK-SD-NEXT: ld1 { v21.b }[4], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[4], [x9]
; CHECK-SD-NEXT: add x9, sp, #912
-; CHECK-SD-NEXT: ld1 { v23.b }[4], [x13]
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x10]
-; CHECK-SD-NEXT: add x10, sp, #848
-; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11]
-; CHECK-SD-NEXT: ld1 { v19.b }[4], [x12]
-; CHECK-SD-NEXT: add x11, sp, #584
-; CHECK-SD-NEXT: add x12, sp, #648
-; CHECK-SD-NEXT: mov v3.b[5], w5
-; CHECK-SD-NEXT: ld1 { v22.b }[5], [x9]
-; CHECK-SD-NEXT: ld1 { v23.b }[5], [x12]
-; CHECK-SD-NEXT: add x12, sp, #592
-; CHECK-SD-NEXT: movi v21.2d, #0000000000000000
+; CHECK-SD-NEXT: add x10, sp, #648
+; CHECK-SD-NEXT: mov v2.b[5], w5
+; CHECK-SD-NEXT: ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v20.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #56
+; CHECK-SD-NEXT: add x8, sp, #192
+; CHECK-SD-NEXT: ld1 { v21.b }[5], [x11]
; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #856
-; CHECK-SD-NEXT: add x9, sp, #920
-; CHECK-SD-NEXT: add x13, sp, #656
-; CHECK-SD-NEXT: add x10, sp, #64
-; CHECK-SD-NEXT: ld1 { v22.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v23.b }[6], [x13]
-; CHECK-SD-NEXT: mov v3.b[6], w6
-; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT: ld1 { v19.b }[6], [x12]
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #864
-; CHECK-SD-NEXT: add x11, sp, #600
-; CHECK-SD-NEXT: add x9, sp, #928
-; CHECK-SD-NEXT: add x12, sp, #664
-; CHECK-SD-NEXT: mov v21.s[0], v24.s[0]
-; CHECK-SD-NEXT: ld1 { v22.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v19.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v23.b }[7], [x12]
+; CHECK-SD-NEXT: add x10, sp, #920
+; CHECK-SD-NEXT: add x11, sp, #656
+; CHECK-SD-NEXT: ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT: sshll v22.4s, v22.4h, #0
+; CHECK-SD-NEXT: ld1 { v20.b }[6], [x11]
+; CHECK-SD-NEXT: mov v2.b[6], w6
+; CHECK-SD-NEXT: add x9, sp, #328
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #64
+; CHECK-SD-NEXT: ld1 { v21.b }[6], [x9]
+; CHECK-SD-NEXT: ld1 { v18.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #928
+; CHECK-SD-NEXT: add x9, sp, #664
+; CHECK-SD-NEXT: ld1 { v19.b }[7], [x8]
+; CHECK-SD-NEXT: ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT: fmov s22, s22
+; CHECK-SD-NEXT: mov v2.b[7], w7
; CHECK-SD-NEXT: add x8, sp, #200
-; CHECK-SD-NEXT: mov v3.b[7], w7
-; CHECK-SD-NEXT: add x10, sp, #336
+; CHECK-SD-NEXT: add x9, sp, #336
+; CHECK-SD-NEXT: add x10, sp, #72
; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #72
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x10]
-; CHECK-SD-NEXT: smull v18.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8]
-; CHECK-SD-NEXT: smull v22.8h, v23.8b, v22.8b
-; CHECK-SD-NEXT: sshll v20.4s, v20.4h, #0
+; CHECK-SD-NEXT: ld1 { v21.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10]
+; CHECK-SD-NEXT: smull v19.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT: sshll v16.4s, v16.4h, #0
; CHECK-SD-NEXT: smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT: saddw v1.4s, v21.4s, v16.4h
-; CHECK-SD-NEXT: smull v2.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT: saddw v1.4s, v22.4s, v6.4h
+; CHECK-SD-NEXT: smull v2.8h, v2.8b, v3.8b
; CHECK-SD-NEXT: smull v3.8h, v5.8b, v4.8b
-; CHECK-SD-NEXT: smull v4.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT: mov v19.s[0], v20.s[0]
-; CHECK-SD-NEXT: saddl2 v5.4s, v18.8h, v17.8h
-; CHECK-SD-NEXT: saddl v7.4s, v18.4h, v17.4h
-; CHECK-SD-NEXT: saddl2 v6.4s, v16.8h, v22.8h
-; CHECK-SD-NEXT: saddw v1.4s, v1.4s, v22.4h
-; CHECK-SD-NEXT: saddl2 v17.4s, v2.8h, v0.8h
-; CHECK-SD-NEXT: saddl2 v16.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT: saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT: saddw v2.4s, v19.4s, v2.4h
-; CHECK-SD-NEXT: add v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT: smull v4.8h, v18.8b, v21.8b
+; CHECK-SD-NEXT: fmov s5, s16
+; CHECK-SD-NEXT: saddl2 v16.4s, v17.8h, v7.8h
+; CHECK-SD-NEXT: saddl2 v6.4s, v6.8h, v19.8h
+; CHECK-SD-NEXT: saddl v7.4s, v17.4h, v7.4h
+; CHECK-SD-NEXT: saddw v1.4s, v1.4s, v19.4h
+; CHECK-SD-NEXT: saddl2 v18.4s, v2.8h, v0.8h
+; CHECK-SD-NEXT: saddl2 v17.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT: saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT: add v5.4s, v6.4s, v16.4s
; CHECK-SD-NEXT: add v1.4s, v1.4s, v7.4s
-; CHECK-SD-NEXT: add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT: saddl v3.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT: add v6.4s, v18.4s, v17.4s
; CHECK-SD-NEXT: saddw v0.4s, v2.4s, v0.4h
; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
; CHECK-SD-NEXT: add v0.4s, v0.4s, v3.4s
@@ -6577,146 +6558,144 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
; CHECK-SD-NEXT: ldr b0, [sp, #80]
; CHECK-SD-NEXT: add x8, sp, #88
; CHECK-SD-NEXT: ldr b2, [sp, #144]
-; CHECK-SD-NEXT: add x9, sp, #152
+; CHECK-SD-NEXT: add x9, sp, #96
; CHECK-SD-NEXT: ldr b3, [sp, #16]
-; CHECK-SD-NEXT: add x11, sp, #104
+; CHECK-SD-NEXT: add x11, sp, #112
; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #24
-; CHECK-SD-NEXT: add x8, sp, #96
-; CHECK-SD-NEXT: ld1 { v3.b }[1], [x9]
-; CHECK-SD-NEXT: ldr b5, [sp, #480]
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: add x10, sp, #112
+; CHECK-SD-NEXT: add x8, sp, #152
; CHECK-SD-NEXT: add x12, sp, #168
-; CHECK-SD-NEXT: ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #160
+; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #24
; CHECK-SD-NEXT: ldr b4, [sp, #608]
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8]
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #104
+; CHECK-SD-NEXT: fmov s1, w0
+; CHECK-SD-NEXT: ld1 { v0.b }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: ldr b5, [sp, #480]
+; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT: add x13, sp, #184
+; CHECK-SD-NEXT: ldr b6, [sp, #672]
+; CHECK-SD-NEXT: mov v1.b[1], w1
+; CHECK-SD-NEXT: ldr b7, [sp, #544]
+; CHECK-SD-NEXT: add x10, sp, #120
+; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
; CHECK-SD-NEXT: add x8, sp, #32
-; CHECK-SD-NEXT: add x13, sp, #496
+; CHECK-SD-NEXT: add x9, sp, #128
; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
-; CHECK-SD-NEXT: mov v1.b[1], w1
-; CHECK-SD-NEXT: ldr b6, [sp, #672]
-; CHECK-SD-NEXT: ld1 { v0.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #488
-; CHECK-SD-NEXT: add x9, sp, #120
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #40
; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12]
-; CHECK-SD-NEXT: ld1 { v3.b }[3], [x11]
-; CHECK-SD-NEXT: add x12, sp, #616
-; CHECK-SD-NEXT: ldr b16, [sp, #544]
-; CHECK-SD-NEXT: ld1 { v0.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #48
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x12]
; CHECK-SD-NEXT: add x12, sp, #176
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT: add x13, sp, #680
-; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT: add x14, sp, #56
+; CHECK-SD-NEXT: add x15, sp, #560
+; CHECK-SD-NEXT: ldr b17, [sp, #736]
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x11]
+; CHECK-SD-NEXT: add x11, sp, #40
+; CHECK-SD-NEXT: mov v1.b[2], w2
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #616
; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT: add x12, sp, #48
+; CHECK-SD-NEXT: add x11, sp, #192
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT: sshll v17.8h, v17.8b, #0
+; CHECK-SD-NEXT: ldr b16, [sp, #208]
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT: add x12, sp, #624
+; CHECK-SD-NEXT: ld1 { v2.b }[5], [x13]
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12]
+; CHECK-SD-NEXT: add x12, sp, #488
+; CHECK-SD-NEXT: add x13, sp, #680
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x12]
; CHECK-SD-NEXT: ld1 { v6.b }[1], [x13]
-; CHECK-SD-NEXT: add x13, sp, #56
-; CHECK-SD-NEXT: ld1 { v0.b }[5], [x9]
-; CHECK-SD-NEXT: mov v1.b[2], w2
-; CHECK-SD-NEXT: add x8, sp, #128
-; CHECK-SD-NEXT: add x14, sp, #184
-; CHECK-SD-NEXT: add x11, sp, #136
-; CHECK-SD-NEXT: ld1 { v3.b }[5], [x13]
; CHECK-SD-NEXT: add x13, sp, #552
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x14]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x13]
-; CHECK-SD-NEXT: add x14, sp, #624
-; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8]
-; CHECK-SD-NEXT: add x8, sp, #688
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT: add x13, sp, #496
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x14]
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT: add x9, sp, #688
+; CHECK-SD-NEXT: mov v1.b[3], w3
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
; CHECK-SD-NEXT: add x13, sp, #504
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x14]
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #560
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x15]
+; CHECK-SD-NEXT: add x12, sp, #64
+; CHECK-SD-NEXT: add x14, sp, #632
+; CHECK-SD-NEXT: ld1 { v2.b }[6], [x11]
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x14]
; CHECK-SD-NEXT: ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x8]
-; CHECK-SD-NEXT: mov v1.b[3], w3
-; CHECK-SD-NEXT: add x9, sp, #64
-; CHECK-SD-NEXT: add x15, sp, #632
-; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v0.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x15]
-; CHECK-SD-NEXT: add x8, sp, #696
-; CHECK-SD-NEXT: add x9, sp, #568
-; CHECK-SD-NEXT: add x11, sp, #512
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT: add x8, sp, #640
+; CHECK-SD-NEXT: add x9, sp, #696
+; CHECK-SD-NEXT: add x11, sp, #568
+; CHECK-SD-NEXT: add x12, sp, #512
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x11]
+; CHECK-SD-NEXT: add x9, sp, #640
; CHECK-SD-NEXT: mov v1.b[4], w4
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #704
-; CHECK-SD-NEXT: add x9, sp, #576
-; CHECK-SD-NEXT: add x11, sp, #520
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT: ldr b18, [sp, #736]
-; CHECK-SD-NEXT: add x12, sp, #192
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x12]
-; CHECK-SD-NEXT: add x8, sp, #648
-; CHECK-SD-NEXT: add x9, sp, #528
-; CHECK-SD-NEXT: add x11, sp, #712
-; CHECK-SD-NEXT: add x12, sp, #584
-; CHECK-SD-NEXT: sshll v18.8h, v18.8b, #0
+; CHECK-SD-NEXT: add x11, sp, #576
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9]
+; CHECK-SD-NEXT: add x9, sp, #704
+; CHECK-SD-NEXT: add x12, sp, #520
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x11]
+; CHECK-SD-NEXT: add x9, sp, #648
+; CHECK-SD-NEXT: add x11, sp, #528
+; CHECK-SD-NEXT: add x13, sp, #584
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x12]
+; CHECK-SD-NEXT: add x12, sp, #712
; CHECK-SD-NEXT: mov v1.b[5], w5
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x12]
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT: add x8, sp, #656
-; CHECK-SD-NEXT: add x9, sp, #536
-; CHECK-SD-NEXT: add x11, sp, #720
-; CHECK-SD-NEXT: add x12, sp, #592
-; CHECK-SD-NEXT: sshll v18.4s, v18.4h, #0
-; CHECK-SD-NEXT: ldr b7, [sp, #208]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x12]
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #656
+; CHECK-SD-NEXT: add x12, sp, #720
+; CHECK-SD-NEXT: add x13, sp, #592
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT: add x11, sp, #536
+; CHECK-SD-NEXT: sshll v17.4s, v17.4h, #0
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x12]
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x13]
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x9]
; CHECK-SD-NEXT: mov v1.b[6], w6
-; CHECK-SD-NEXT: sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT: add x8, sp, #664
-; CHECK-SD-NEXT: add x9, sp, #728
-; CHECK-SD-NEXT: add x11, sp, #600
-; CHECK-SD-NEXT: mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT: sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT: add x9, sp, #664
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x11]
+; CHECK-SD-NEXT: add x11, sp, #728
+; CHECK-SD-NEXT: add x12, sp, #600
+; CHECK-SD-NEXT: fmov s17, s17
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT: add x8, sp, #136
; CHECK-SD-NEXT: add x10, sp, #200
+; CHECK-SD-NEXT: sshll v5.8h, v5.8b, #0
; CHECK-SD-NEXT: mov v1.b[7], w7
-; CHECK-SD-NEXT: add x9, sp, #72
-; CHECK-SD-NEXT: sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT: add x11, sp, #72
+; CHECK-SD-NEXT: sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT: ld1 { v0.b }[7], [x8]
; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11]
; CHECK-SD-NEXT: sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT: sshll v7.8h, v7.8b, #0
; CHECK-SD-NEXT: sshll v4.8h, v4.8b, #0
; CHECK-SD-NEXT: saddw v17.4s, v17.4s, v5.4h
+; CHECK-SD-NEXT: fmov s16, s16
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT: mov v18.s[0], v7.s[0]
; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-SD-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-SD-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT: saddl2 v7.4s, v16.8h, v6.8h
+; CHECK-SD-NEXT: saddl2 v18.4s, v7.8h, v6.8h
; CHECK-SD-NEXT: saddl2 v5.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT: saddl v6.4s, v16.4h, v6.4h
+; CHECK-SD-NEXT: saddl v6.4s, v7.4h, v6.4h
; CHECK-SD-NEXT: saddw v4.4s, v17.4s, v4.4h
; CHECK-SD-NEXT: saddl2 v17.4s, v1.8h, v0.8h
-; CHECK-SD-NEXT: saddl2 v16.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT: saddw v1.4s, v18.4s, v1.4h
-; CHECK-SD-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-SD-NEXT: add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT: saddl2 v7.4s, v3.8h, v2.8h
+; CHECK-SD-NEXT: saddw v1.4s, v16.4s, v1.4h
; CHECK-SD-NEXT: saddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT: add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT: add v5.4s, v5.4s, v18.4s
+; CHECK-SD-NEXT: add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT: add v6.4s, v17.4s, v7.4s
; CHECK-SD-NEXT: saddw v0.4s, v1.4s, v0.4h
; CHECK-SD-NEXT: add v1.4s, v4.4s, v5.4s
; CHECK-SD-NEXT: add v0.4s, v0.4s, v2.4s
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index 07dc35d8ce6ee0..f0aa700c599ab0 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -60,9 +60,7 @@ entry:
define <4 x float> @testv4f32_canonical(<4 x float> noundef %vec) {
; CHECK-SD-LABEL: testv4f32_canonical:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: fmov s0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4f32_canonical:
@@ -78,12 +76,17 @@ entry:
}
define <4 x float> @testv4f32_0(<4 x float> noundef %vec) {
-; CHECK-LABEL: testv4f32_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: testv4f32_0:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: fmov s0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4f32_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x float> %vec, <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
ret <4 x float> %vecinit3
@@ -92,9 +95,7 @@ entry:
define <4 x float> @testv4f32_1(<4 x float> noundef %vec) {
; CHECK-SD-LABEL: testv4f32_1:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: fmov s0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4f32_1:
@@ -166,12 +167,12 @@ entry:
}
define <8 x half> @testv8f16_canonical(<8 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv8f16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8f16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv8f16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -181,18 +182,35 @@ define <8 x half> @testv8f16_canonical(<8 x half> noundef %vec) {
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI9_0]
; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8f16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x half> %vec, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
ret <8 x half> %vecinit3
}
define <8 x half> @testv8f16_0(<8 x half> noundef %vec) {
-; CHECK-LABEL: testv8f16_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8f16_0:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8f16_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8f16_0:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x half> %vec, <8 x half> <half poison, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
ret <8 x half> %vecinit3
@@ -220,13 +238,13 @@ entry:
}
define <4 x half> @testv4f16_canonical(<4 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv4f16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: fmov d0, d1
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv4f16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: fmov d0, d1
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4f16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -236,6 +254,11 @@ define <4 x half> @testv4f16_canonical(<4 x half> noundef %vec) {
; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
; CHECK-GI-NEXT: fmov d0, d1
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv4f16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x half> %vec, <4 x half> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
ret <4 x half> %vecinit3
@@ -285,12 +308,12 @@ entry:
}
define <8 x bfloat> @testv8bf16_canonical(<8 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv8bf16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8bf16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv8bf16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -300,18 +323,35 @@ define <8 x bfloat> @testv8bf16_canonical(<8 x bfloat> noundef %vec) {
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI15_0]
; CHECK-GI-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8bf16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
ret <8 x bfloat> %vecinit3
}
define <8 x bfloat> @testv8bf16_0(<8 x bfloat> noundef %vec) {
-; CHECK-LABEL: testv8bf16_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8bf16_0:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8bf16_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8bf16_0:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> <bfloat poison, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
ret <8 x bfloat> %vecinit3
@@ -339,13 +379,13 @@ entry:
}
define <4 x bfloat> @testv4bf16_canonical(<4 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv4bf16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: fmov d0, d1
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv4bf16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: fmov d0, d1
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4bf16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -355,6 +395,11 @@ define <4 x bfloat> @testv4bf16_canonical(<4 x bfloat> noundef %vec) {
; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
; CHECK-GI-NEXT: fmov d0, d1
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv4bf16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x bfloat> %vec, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
ret <4 x bfloat> %vecinit3
@@ -455,9 +500,7 @@ entry:
define <4 x i32> @testv4i32_canonical(<4 x i32> noundef %vec) {
; CHECK-SD-LABEL: testv4i32_canonical:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: fmov s0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4i32_canonical:
@@ -473,12 +516,17 @@ entry:
}
define <4 x i32> @testv4i32_0(<4 x i32> noundef %vec) {
-; CHECK-LABEL: testv4i32_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: testv4i32_0:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: fmov s0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4i32_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x i32> %vec, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
ret <4 x i32> %vecinit3
@@ -487,9 +535,7 @@ entry:
define <4 x i32> @testv4i32_1(<4 x i32> noundef %vec) {
; CHECK-SD-LABEL: testv4i32_1:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-SD-NEXT: fmov s0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4i32_1:
@@ -560,12 +606,12 @@ entry:
}
define <8 x i16> @testv8i16_canonical(<8 x i16> noundef %vec) {
-; CHECK-SD-LABEL: testv8i16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8i16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv8i16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -573,18 +619,35 @@ define <8 x i16> @testv8i16_canonical(<8 x i16> noundef %vec) {
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI30_0]
; CHECK-GI-NEXT: tbl v0.16b, { v0.16b }, v1.16b
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8i16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
ret <8 x i16> %vecinit3
}
define <8 x i16> @testv8i16_0(<8 x i16> noundef %vec) {
-; CHECK-LABEL: testv8i16_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv8i16_0:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: mov v0.16b, v1.16b
+; CHECK-NO-FP16-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv8i16_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv8i16_0:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> <i16 poison, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
ret <8 x i16> %vecinit3
@@ -603,13 +666,13 @@ entry:
}
define <4 x i16> @testv4i16_canonical(<4 x i16> noundef %vec) {
-; CHECK-SD-LABEL: testv4i16_canonical:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT: fmov d0, d1
-; CHECK-SD-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv4i16_canonical:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: fmov d0, d1
+; CHECK-NO-FP16-SD-NEXT: ret
;
; CHECK-GI-LABEL: testv4i16_canonical:
; CHECK-GI: // %bb.0: // %entry
@@ -619,19 +682,37 @@ define <4 x i16> @testv4i16_canonical(<4 x i16> noundef %vec) {
; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
; CHECK-GI-NEXT: fmov d0, d1
; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv4i16_canonical:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
ret <4 x i16> %vecinit3
}
define <4 x i16> @testv4i16_0(<4 x i16> noundef %vec) {
-; CHECK-LABEL: testv4i16_0:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v1.h[0], v0.h[0]
-; CHECK-NEXT: fmov d0, d1
-; CHECK-NEXT: ret
+; CHECK-NO-FP16-SD-LABEL: testv4i16_0:
+; CHECK-NO-FP16-SD: // %bb.0: // %entry
+; CHECK-NO-FP16-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NO-FP16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NO-FP16-SD-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-NO-FP16-SD-NEXT: fmov d0, d1
+; CHECK-NO-FP16-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testv4i16_0:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: ret
+;
+; CHECK-FP16-SD-LABEL: testv4i16_0:
+; CHECK-FP16-SD: // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT: fmov h0, h0
+; CHECK-FP16-SD-NEXT: ret
entry:
%vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> <i16 poison, i16 0, i16 0, i16 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
ret <4 x i16> %vecinit3
@@ -767,6 +848,4 @@ entry:
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK-FP16-GI: {{.*}}
-; CHECK-FP16-SD: {{.*}}
; CHECK-NO-FP16-GI: {{.*}}
-; CHECK-NO-FP16-SD: {{.*}}
More information about the llvm-branch-commits
mailing list