[llvm-branch-commits] [llvm] [AArch64][DAG] Improve codegen for clearing upper vector lanes (PR #237259)

Kieran B via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Tue Nov 3 01:18:50 PST 2026


https://github.com/kieroxide updated https://github.com/llvm/llvm-project/pull/237259

>From d831651978ef95aad446f1025109149e51dd52f1 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:15 +0000
Subject: [PATCH 1/2] [AArch64][DAG] Improve codegen for clearing upper vector
 lanes

Add patterns that use scalar fmov instruction to clear the upper lanes of a vector
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  49 ++
 .../CodeGen/AArch64/shuffle-insert-zero.ll    | 633 +++++++-----------
 2 files changed, 276 insertions(+), 406 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e02132182e7eb..5bf6613019527 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8759,6 +8759,55 @@ defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v4i16, scalar_to_vector, INSvi8lane
 defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v2i32, scalar_to_vector_v2f32, INSvi8lane, DUPi8, bsub, VecIndex_x4>;
 defm : Neon_INS_elt_ext_pattern<v8i16, v4i16, v2i32, scalar_to_vector_v2f32, INSvi16lane, DUPi16, hsub, VecIndex_x2>;
 
+multiclass ExtractInsertLane0IntoZeroVector<ValueType VecVT, ValueType ElemTy> {
+  let Predicates = [HasFullFP16] in {
+    def : Pat<(VecVT (vector_insert immAllZerosV,
+                (ElemTy (vector_extract (VecVT V128:$Rn), (i64 0))), (i64 0))),
+              (VecVT (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (VecVT V128:$Rn), hsub))), hsub))>;
+  }
+}
+
+defm : ExtractInsertLane0IntoZeroVector<v8i16,  i32>;
+defm : ExtractInsertLane0IntoZeroVector<v8f16,  f16>;
+defm : ExtractInsertLane0IntoZeroVector<v8bf16, bf16>;
+
+let Predicates = [HasFullFP16] in {
+  def : Pat<(v4i16 (vector_insert immAllZerosV, (i32 (vector_extract
+                (v8i16 (insert_subvector (v8i16 undef), (v4i16 V64:$Rn), (i64 0))),
+                (i64 0))),
+              (i64 0))),
+            (v4i16 (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (v4i16 V64:$Rn), hsub))), hsub))>;
+
+  def : Pat<(v4i16 (vector_insert immAllZerosV,
+            (i32 (vector_extract
+                (v8i16 (bitconvert
+                  (v8f16 (insert_subvector (v8f16 undef),
+                    (v4f16 V64:$Rn), (i64 0))))),
+                (i64 0))),
+              (i64 0))),
+            (v4i16 (SUBREG_TO_REG
+              (FMOVHr (f16 (EXTRACT_SUBREG (v4f16 V64:$Rn), hsub))),
+              hsub))>;
+
+  def : Pat<(v4i16 (vector_insert immAllZerosV,
+              (i32 (vector_extract
+                (v8i16 (bitconvert
+                  (v8bf16 (insert_subvector (v8bf16 undef),
+                    (v4bf16 V64:$Rn), (i64 0))))),
+                (i64 0))),
+              (i64 0))),
+            (v4i16 (SUBREG_TO_REG
+              (FMOVHr (f16 (EXTRACT_SUBREG (v4bf16 V64:$Rn), hsub))),
+              hsub))>;
+}
+
+// extract from lane 0 and insert into lane 0 of a zero vector
+// is equivalent to clearing upper bits of a register, we can use fmov
+def : Pat<(v4i32 (vector_insert immAllZerosV, (i32 (vector_extract (v4i32 V128:$Rn), (i64 0))), (i64 0))),
+          (v4i32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4i32 V128:$Rn), ssub)), ssub))>;
+def : Pat<(v4f32 (vector_insert immAllZerosV, (f32 (vector_extract (v4f32 V128:$Rn), (i64 0))), (i64 0))),
+          (v4f32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4f32 V128:$Rn), ssub)), ssub))>;
+
 // bitcast of an extract
 // f32 bitcast(vector_extract(v4i32 src, 0)) -> EXTRACT_SUBREG(src)
 def : Pat<(f32 (bitconvert (i32 (vector_extract v16i8:$src, (i64 0))))),
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index 30bac9297abea..dab16a39802df 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -1,483 +1,304 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -O2 | FileCheck %s --check-prefixes=CHECK,SD,SD-FP16
+; RUN: llc < %s -mtriple=aarch64 -O2 | FileCheck %s --check-prefixes=CHECK,SD,SD-NOFP16
+; RUN: llc < %s -mtriple=aarch64 -O2 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,GI,GI-FP16
+; RUN: llc < %s -mtriple=aarch64 -O2 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,GI,GI-NOFP16
 
+; The canonical form uses <i32 0, i32 poison, i32 poison, ... > for the zero vector operand
+; We use equivalent zeroinitializer instead which has no functional change
 
-define <2 x double> @testv2f64_0(<2 x double> noundef %vec) {
-; CHECK-SD-LABEL: testv2f64_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+define <2 x double> @testv2f64(<2 x double> noundef %vec) {
+; SD-LABEL: testv2f64:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f64_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.d[1], v1.d[0]
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f64:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
+; GI-NEXT:    ret
 entry:
-  %vecinit1 = shufflevector <2 x double> %vec, <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+  %vecinit1 = shufflevector <2 x double> %vec, <2 x double> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ret <2 x double> %vecinit1
 }
 
-define <2 x double> @testv2f64_1(<2 x double> noundef %vec) {
-; CHECK-SD-LABEL: testv2f64_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT:    ret
+define <4 x float> @testv4f32(<4 x float> noundef %vec) {
+; SD-LABEL: testv4f32:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.s[0], v0.s[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f64_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4f32:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    dup v1.4s, v1.s[0]
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
-  %vecins = shufflevector <2 x double> <double 0.000000e+00, double poison>, <2 x double> %vec, <2 x i32> <i32 2, i32 0>
-  ret <2 x double> %vecins
-}
-
-define <4 x float> @testv4f32_0(<4 x float> noundef %vec) {
-; CHECK-LABEL: testv4f32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
-entry:
-  %vecinit3 = shufflevector <4 x float> %vec, <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  %vecinit3 = shufflevector <4 x float> %vec, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
   ret <4 x float> %vecinit3
 }
 
-define <4 x float> @testv4f32_1(<4 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv4f32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv4f32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, <4 x float> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
-  ret <4 x float> %vecins
-}
 
-define <2 x float> @testv2f32_0(<2 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv2f32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+define <2 x float> @testv2f32(<2 x float> noundef %vec) {
+; SD-LABEL: testv2f32:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.s[1], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f32:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <2 x float> %vec, <2 x float> <float poison, float 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+  %vecinit3 = shufflevector <2 x float> %vec, <2 x float> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ret <2 x float> %vecinit3
 }
 
-define <2 x float> @testv2f32_1(<2 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv2f32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv2f32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <2 x float> <float 0.000000e+00, float poison>, <2 x float> %vec, <2 x i32> <i32 2, i32 0>
-  ret <2 x float> %vecins
-}
 
-define <8 x half> @testv8f16_0(<8 x half> noundef %vec) {
-; CHECK-LABEL: testv8f16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+define <8 x half> @testv8f16(<8 x half> noundef %vec) {
+; SD-LABEL: testv8f16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv8f16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; GI-NEXT:    adrp x8, .LCPI3_0
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <8 x half> %vec, <8 x half> <half poison, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vecinit3 = shufflevector <8 x half> %vec, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x half> %vecinit3
 }
 
-define <8 x half> @testv8f16_1(<8 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv8f16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    adrp x8, .LCPI7_0
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv8f16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI7_0
-; CHECK-GI-NEXT:    movi d31, #0000000000000000
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <8 x half> <half 0.000000e+00, half poison, half poison, half poison, half poison, half poison, half poison, half poison>, <8 x half> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-  ret <8 x half> %vecins
-}
 
-define <4 x half> @testv4f16_0(<4 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv4f16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+define <4 x half> @testv4f16(<4 x half> noundef %vec) {
+; SD-LABEL: testv4f16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4f16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4f16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <4 x half> %vec, <4 x half> <half poison, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  %vecinit3 = shufflevector <4 x half> %vec, <4 x half> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
   ret <4 x half> %vecinit3
 }
 
-define <4 x half> @testv4f16_1(<4 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv4f16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv4f16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <4 x half> <half 0.000000e+00, half poison, half poison, half poison>, <4 x half> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
-  ret <4 x half> %vecins
-}
 
-define <8 x bfloat> @testv8bf16_0(<8 x bfloat> noundef %vec) {
-; CHECK-LABEL: testv8bf16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+define <8 x bfloat> @testv8bf16(<8 x bfloat> noundef %vec) {
+; SD-LABEL: testv8bf16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv8bf16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    // kill: def $q0 killed $q0 def $q0_q1
+; GI-NEXT:    adrp x8, .LCPI5_0
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    ldr q2, [x8, :lo12:.LCPI5_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> <bfloat poison, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x bfloat> %vecinit3
 }
 
-define <8 x bfloat> @testv8bf16_1(<8 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv8bf16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    adrp x8, .LCPI11_0
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI11_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv8bf16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI11_0
-; CHECK-GI-NEXT:    movi d31, #0000000000000000
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI11_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <8 x bfloat> <bfloat 0.000000e+00, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison>, <8 x bfloat> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-  ret <8 x bfloat> %vecins
-}
 
-define <4 x bfloat> @testv4bf16_0(<4 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv4bf16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+define <4 x bfloat> @testv4bf16(<4 x bfloat> noundef %vec) {
+; SD-LABEL: testv4bf16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4bf16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4bf16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <4 x bfloat> %vec, <4 x bfloat> <bfloat poison, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  %vecinit3 = shufflevector <4 x bfloat> %vec, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
   ret <4 x bfloat> %vecinit3
 }
 
-define <4 x bfloat> @testv4bf16_1(<4 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv4bf16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv4bf16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <4 x bfloat> <bfloat 0.000000e+00, bfloat poison, bfloat poison, bfloat poison>, <4 x bfloat> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
-  ret <4 x bfloat> %vecins
-}
-
-
-define <2 x i64> @testv2i64_0(<2 x i64> noundef %vec) {
-; CHECK-SD-LABEL: testv2i64_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+define <2 x i64> @testv2i64(<2 x i64> noundef %vec) {
+; SD-LABEL: testv2i64:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i64_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i64:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
+; GI-NEXT:    ret
 entry:
-  %vecinit1 = shufflevector <2 x i64> %vec, <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
+  %vecinit1 = shufflevector <2 x i64> %vec, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ret <2 x i64> %vecinit1
 }
 
-define <2 x i64> @testv2i64_1(<2 x i64> noundef %vec) {
-; CHECK-SD-LABEL: testv2i64_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv2i64_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <2 x i64> <i64 0, i64 poison>, <2 x i64> %vec, <2 x i32> <i32 2, i32 0>
-  ret <2 x i64> %vecins
-}
 
-define <4 x i32> @testv4i32_0(<4 x i32> noundef %vec) {
-; CHECK-LABEL: testv4i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+define <4 x i32> @testv4i32(<4 x i32> noundef %vec) {
+; SD-LABEL: testv4i32:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.s[0], v0.s[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv4i32:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    dup v1.4s, v1.s[0]
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <4 x i32> %vec, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  %vecinit3 = shufflevector <4 x i32> %vec, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
   ret <4 x i32> %vecinit3
 }
 
-define <4 x i32> @testv4i32_1(<4 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv4i32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv4i32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, <4 x i32> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
-  ret <4 x i32> %vecins
-}
 
-define <2 x i32> @testv2i32_0(<2 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv2i32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+define <2 x i32> @testv2i32(<2 x i32> noundef %vec) {
+; SD-LABEL: testv2i32:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[1], wzr
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i32:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <2 x i32> %vec, <2 x i32> <i32 poison, i32 0>, <2 x i32> <i32 0, i32 3>
+  %vecinit3 = shufflevector <2 x i32> %vec, <2 x i32> zeroinitializer, <2 x i32> <i32 0, i32 2>
   ret <2 x i32> %vecinit3
 }
 
-define <2 x i32> @testv2i32_1(<2 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv2i32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv2i32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <2 x i32> <i32 0, i32 poison>, <2 x i32> %vec, <2 x i32> <i32 2, i32 0>
-  ret <2 x i32> %vecins
-}
 
-define <8 x i16> @testv8i16_0(<8 x i16> noundef %vec) {
-; CHECK-LABEL: testv8i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+define <8 x i16> @testv8i16(<8 x i16> noundef %vec) {
+; SD-LABEL: testv8i16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv8i16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    adrp x8, .LCPI10_0
+; GI-NEXT:    ldr q1, [x8, :lo12:.LCPI10_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> <i16 poison, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i16> %vecinit3
 }
 
-define <8 x i16> @testv8i16_1(<8 x i16> noundef %vec) {
-; CHECK-LABEL: testv8i16_1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adrp x8, .LCPI21_0
-; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
-; CHECK-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-NEXT:    ret
-entry:
-  %vecins = shufflevector <8 x i16> <i16 0, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>, <8 x i16> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-  ret <8 x i16> %vecins
-}
 
-define <4 x i16> @testv4i16_0(<4 x i16> noundef %vec) {
-; CHECK-LABEL: testv4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    fmov d0, d1
-; CHECK-NEXT:    ret
+define <4 x i16> @testv4i16(<4 x i16> noundef %vec) {
+; SD-LABEL: testv4i16:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv4i16:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> <i16 poison, i16 0, i16 0, i16 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  %vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> zeroinitializer, <4 x i32> <i32 0, i32 4, i32 4, i32 4>
   ret <4 x i16> %vecinit3
 }
 
-define <4 x i16> @testv4i16_1(<4 x i16> noundef %vec) {
-; CHECK-SD-LABEL: testv4i16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv4i16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <4 x i16> <i16 0, i16 poison, i16 poison, i16 poison>, <4 x i16> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
-  ret <4 x i16> %vecins
-}
 
-define <16 x i8> @testv16i8_0(<16 x i8> noundef %vec) {
-; CHECK-LABEL: testv16i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.b[0], v0.b[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+define <16 x i8> @testv16i8(<16 x i8> noundef %vec) {
+; SD-LABEL: testv16i8:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.b[0], v0.b[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv16i8:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    adrp x8, .LCPI12_0
+; GI-NEXT:    ldr q1, [x8, :lo12:.LCPI12_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <16 x i8> %vec, <16 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %vecinit3 = shufflevector <16 x i8> %vec, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
   ret <16 x i8> %vecinit3
 }
 
-define <16 x i8> @testv16i8_1(<16 x i8> noundef %vec) {
-; CHECK-LABEL: testv16i8_1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adrp x8, .LCPI25_0
-; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI25_0]
-; CHECK-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-NEXT:    ret
-entry:
-  %vecins = shufflevector <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i8> %vec, <16 x i32> <i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-  ret <16 x i8> %vecins
-}
 
-define <8 x i8> @testv8i8_0(<8 x i8> noundef %vec) {
-; CHECK-LABEL: testv8i8_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov v1.b[0], v0.b[0]
-; CHECK-NEXT:    fmov d0, d1
-; CHECK-NEXT:    ret
+define <8 x i8> @testv8i8(<8 x i8> noundef %vec) {
+; SD-LABEL: testv8i8:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.b[0], v0.b[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv8i8:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    adrp x8, .LCPI13_0
+; GI-NEXT:    mov v0.d[1], v1.d[0]
+; GI-NEXT:    ldr d1, [x8, :lo12:.LCPI13_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; GI-NEXT:    ret
 entry:
-  %vecinit3 = shufflevector <8 x i8> %vec, <8 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vecinit3 = shufflevector <8 x i8> %vec, <8 x i8> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
   ret <8 x i8> %vecinit3
 }
 
-define <8 x i8> @testv8i8_1(<8 x i8> noundef %vec) {
-; CHECK-SD-LABEL: testv8i8_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    movi d1, #0xffffffffffffff00
-; CHECK-SD-NEXT:    mov v0.d[1], v0.d[0]
-; CHECK-SD-NEXT:    tbl v0.8b, { v0.16b }, v1.8b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: testv8i8_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI27_0
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI27_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
-entry:
-  %vecins = shufflevector <8 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <8 x i8> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
-  ret <8 x i8> %vecins
-}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
+; GI-FP16: {{.*}}
+; GI-NOFP16: {{.*}}
+; SD-FP16: {{.*}}
+; SD-NOFP16: {{.*}}

>From 25ef8e1052f4af542356d38495656d0843f3661d Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:32 +0000
Subject: [PATCH 2/2] Update Test Checks

---
 .../aarch64-neon-vector-insert-uaddlv.ll      |   30 +-
 .../CodeGen/AArch64/build-vector-extract.ll   |    4 +-
 llvm/test/CodeGen/AArch64/neon-dotreduce.ll   | 1289 ++++++++---------
 .../CodeGen/AArch64/shuffle-insert-zero.ll    |  120 +-
 4 files changed, 720 insertions(+), 723 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d552397..ad6fc44e1e7fa 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -30,8 +30,8 @@ define void @insert_vec_v4i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v4i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    uaddlv.8h s1, v0
-; CHECK-NEXT:    mov.s v0[0], v1[0]
+; CHECK-NEXT:    uaddlv.8h s0, v0
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ucvtf.4s v0, v0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
@@ -48,11 +48,10 @@ define void @insert_vec_v16i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.8h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
 
@@ -68,14 +67,13 @@ define void @insert_vec_v23i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v23i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    str wzr, [x0, #88]
 ; CHECK-NEXT:    str xzr, [x0, #80]
 ; CHECK-NEXT:    uaddlv.8h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #16]
 ; CHECK-NEXT:    stp q0, q0, [x0, #48]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    str q1, [x0]
 ; CHECK-NEXT:    ret
 
@@ -254,10 +252,9 @@ define void @insert_vec_v16i64_uaddlv_from_v4i16(ptr %0) {
 ; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.4h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    fmov x8, d2
-; CHECK-NEXT:    mov.d x9, v2[1]
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    mov.d x9, v1[1]
 ; CHECK-NEXT:    ucvtf s1, x8
 ; CHECK-NEXT:    ucvtf s3, x9
 ; CHECK-NEXT:    mov.s v2[0], v1[0]
@@ -346,8 +343,8 @@ define void @insert_vec_v8i32_uaddlv_from_v4i32(ptr %0) {
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
 ; CHECK-NEXT:    stp xzr, xzr, [x0, #16]
-; CHECK-NEXT:    uaddlv.4s d1, v0
-; CHECK-NEXT:    mov.s v0[0], v1[0]
+; CHECK-NEXT:    uaddlv.4s d0, v0
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ucvtf.4s v0, v0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
@@ -365,11 +362,10 @@ define void @insert_vec_v16i32_uaddlv_from_v4i32(ptr %0) {
 ; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v4i32:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.4s d1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/AArch64/build-vector-extract.ll b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
index f3e40021b616a..0cfbc939c12a8 100644
--- a/llvm/test/CodeGen/AArch64/build-vector-extract.ll
+++ b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
@@ -15,9 +15,7 @@ define <2 x i64> @extract0_i32_zext_insert0_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; CHECK-LABEL: extract0_i32_zext_insert0_i64_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ret
   %e = extractelement <4 x i32> %x, i32 0
   %z = zext i32 %e to i64
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 2483b476799f6..0fb3cb2643d65 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -321,9 +321,8 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
 ; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -406,10 +405,9 @@ define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v5i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -447,9 +445,8 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -533,14 +530,12 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
 ; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0
-; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT:    mov v1.s[0], v5.s[0]
-; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT:    sshll2 v1.4s, v2.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v2.4h
+; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -682,14 +677,12 @@ define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
 ; CHECK-SD-NEXT:    sshll v1.8h, v2.8b, #0
-; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[0], v5.s[0]
-; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
-; CHECK-SD-NEXT:    saddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    fmov s2, s2
+; CHECK-SD-NEXT:    fmov s3, s3
+; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v3.4s, v1.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -2310,7 +2303,6 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-LABEL: test_udot_v25i8:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldp q3, q0, [x1]
-; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q2, q1, [x0]
 ; CHECK-SD-NEXT:    umull2 v4.8h, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
@@ -2319,10 +2311,10 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    ushll v3.4s, v4.4h, #0
 ; CHECK-SD-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h
 ; CHECK-SD-NEXT:    uaddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uaddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    uaddw v2.4s, v3.4s, v2.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -2637,20 +2629,19 @@ entry:
 define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v25i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q2, q1, [x0]
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-SD-NEXT:    uaddl v1.4s, v4.4h, v1.4h
-; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
-; CHECK-SD-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
-; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    ldp q1, q0, [x0]
+; CHECK-SD-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v4.4s, v3.8h, v0.8h
+; CHECK-SD-NEXT:    uaddl v0.4s, v3.4h, v0.4h
+; CHECK-SD-NEXT:    fmov s2, s2
+; CHECK-SD-NEXT:    uaddw2 v3.4s, v4.4s, v1.8h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT:    uaddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
@@ -2739,7 +2730,6 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-LABEL: test_sdot_v25i8:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldp q3, q0, [x1]
-; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q2, q1, [x0]
 ; CHECK-SD-NEXT:    smull2 v4.8h, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
@@ -2748,10 +2738,10 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    sshll v3.4s, v4.4h, #0
 ; CHECK-SD-NEXT:    saddl2 v4.4s, v1.8h, v0.8h
 ; CHECK-SD-NEXT:    saddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    saddw v2.4s, v3.4s, v2.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -3073,203 +3063,201 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25
 ; CHECK-SD-NEXT:    add x8, sp, #224
 ; CHECK-SD-NEXT:    ldr b1, [sp, #16]
 ; CHECK-SD-NEXT:    ldr b2, [sp, #280]
-; CHECK-SD-NEXT:    add x9, sp, #240
+; CHECK-SD-NEXT:    add x9, sp, #288
 ; CHECK-SD-NEXT:    ldr b4, [sp, #80]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #24
-; CHECK-SD-NEXT:    add x10, sp, #48
+; CHECK-SD-NEXT:    ldr b5, [sp, #152]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #232
-; CHECK-SD-NEXT:    add x11, sp, #96
-; CHECK-SD-NEXT:    ldr b5, [sp, #152]
-; CHECK-SD-NEXT:    add x12, sp, #168
-; CHECK-SD-NEXT:    ldr b6, [sp, #616]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    add x10, sp, #296
+; CHECK-SD-NEXT:    add x11, sp, #64
 ; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #32
-; CHECK-SD-NEXT:    fmov s3, w0
+; CHECK-SD-NEXT:    ldr b6, [sp, #616]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #288
+; CHECK-SD-NEXT:    add x8, sp, #240
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #96
+; CHECK-SD-NEXT:    fmov s3, w0
 ; CHECK-SD-NEXT:    ldr b7, [sp, #416]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #40
-; CHECK-SD-NEXT:    ldr b22, [sp, #744]
-; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #248
-; CHECK-SD-NEXT:    mov v3.b[1], w1
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #88
-; CHECK-SD-NEXT:    ldr b23, [sp, #544]
+; CHECK-SD-NEXT:    ldr b16, [sp, #344]
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x9]
 ; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #256
-; CHECK-SD-NEXT:    ldr b19, [sp, #680]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #296
-; CHECK-SD-NEXT:    ldr b20, [sp, #480]
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    add x10, sp, #160
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #304
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #56
-; CHECK-SD-NEXT:    add x10, sp, #264
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #64
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #248
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    mov v3.b[1], w1
+; CHECK-SD-NEXT:    ldr b17, [sp, #144]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #48
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #256
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    add x8, sp, #264
+; CHECK-SD-NEXT:    ldr b20, [sp, #744]
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #272
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #72
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    ldr b21, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #304
 ; CHECK-SD-NEXT:    mov v3.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #72
+; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ldr b17, [sp, #680]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #104
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #312
 ; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #112
-; CHECK-SD-NEXT:    add x10, sp, #128
+; CHECK-SD-NEXT:    add x8, sp, #320
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x11]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #320
-; CHECK-SD-NEXT:    ldr b21, [sp, #552]
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #176
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #624
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #120
-; CHECK-SD-NEXT:    add x9, sp, #328
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #184
-; CHECK-SD-NEXT:    add x11, sp, #192
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #632
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #624
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x11, sp, #176
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x10]
+; CHECK-SD-NEXT:    add x9, sp, #632
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #328
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #120
+; CHECK-SD-NEXT:    add x11, sp, #184
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #128
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
 ; CHECK-SD-NEXT:    add x8, sp, #336
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #192
+; CHECK-SD-NEXT:    ldr b18, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #200
+; CHECK-SD-NEXT:    ldr b19, [sp, #552]
 ; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #656
-; CHECK-SD-NEXT:    smull v23.8h, v23.8b, v22.8b
+; CHECK-SD-NEXT:    add x8, sp, #648
 ; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #648
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #200
-; CHECK-SD-NEXT:    add x10, sp, #136
-; CHECK-SD-NEXT:    ldr b22, [sp, #352]
-; CHECK-SD-NEXT:    add x12, sp, #360
-; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #424
+; CHECK-SD-NEXT:    smull v21.8h, v21.8b, v20.8b
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #656
+; CHECK-SD-NEXT:    ldr b20, [sp, #352]
 ; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #208
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    add x9, sp, #136
 ; CHECK-SD-NEXT:    add x10, sp, #488
-; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #560
-; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #440
-; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x11]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #432
-; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #208
+; CHECK-SD-NEXT:    add x11, sp, #560
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #440
+; CHECK-SD-NEXT:    add x12, sp, #360
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #696
 ; CHECK-SD-NEXT:    add x11, sp, #496
 ; CHECK-SD-NEXT:    add x12, sp, #568
 ; CHECK-SD-NEXT:    add x13, sp, #368
-; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x11]
-; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x13]
 ; CHECK-SD-NEXT:    add x10, sp, #448
+; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #704
 ; CHECK-SD-NEXT:    mov v3.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #504
 ; CHECK-SD-NEXT:    add x11, sp, #576
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #696
 ; CHECK-SD-NEXT:    add x12, sp, #376
-; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
+; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #712
 ; CHECK-SD-NEXT:    add x11, sp, #512
-; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #504
 ; CHECK-SD-NEXT:    add x12, sp, #584
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #704
-; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
 ; CHECK-SD-NEXT:    add x13, sp, #384
 ; CHECK-SD-NEXT:    mov v3.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x13]
 ; CHECK-SD-NEXT:    add x10, sp, #456
-; CHECK-SD-NEXT:    ldr b16, [sp, #344]
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #712
-; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x11]
-; CHECK-SD-NEXT:    ldr b17, [sp, #144]
+; CHECK-SD-NEXT:    add x9, sp, #720
 ; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
 ; CHECK-SD-NEXT:    add x10, sp, #520
 ; CHECK-SD-NEXT:    add x11, sp, #592
 ; CHECK-SD-NEXT:    add x12, sp, #392
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x12]
 ; CHECK-SD-NEXT:    mov v3.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #720
-; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x12]
-; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    add x9, sp, #728
 ; CHECK-SD-NEXT:    add x8, sp, #664
 ; CHECK-SD-NEXT:    add x10, sp, #464
 ; CHECK-SD-NEXT:    add x11, sp, #528
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #728
 ; CHECK-SD-NEXT:    add x12, sp, #600
+; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x9]
 ; CHECK-SD-NEXT:    add x13, sp, #400
 ; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x13]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #736
 ; CHECK-SD-NEXT:    mov v3.b[7], w7
-; CHECK-SD-NEXT:    sshll v18.4s, v16.4h, #0
-; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
 ; CHECK-SD-NEXT:    add x8, sp, #672
 ; CHECK-SD-NEXT:    add x10, sp, #472
-; CHECK-SD-NEXT:    add x11, sp, #608
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    add x11, sp, #608
 ; CHECK-SD-NEXT:    add x12, sp, #408
-; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x12]
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x12]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
-; CHECK-SD-NEXT:    sshll v23.4s, v23.4h, #0
+; CHECK-SD-NEXT:    sshll v21.4s, v21.4h, #0
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    smull v1.8h, v4.8b, v2.8b
 ; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v5.8b
-; CHECK-SD-NEXT:    smull v3.8h, v20.8b, v19.8b
-; CHECK-SD-NEXT:    smull v4.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT:    smull v5.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT:    mov v16.s[0], v23.s[0]
-; CHECK-SD-NEXT:    saddl2 v6.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    smull v3.8h, v18.8b, v17.8b
+; CHECK-SD-NEXT:    fmov s5, s16
+; CHECK-SD-NEXT:    smull v4.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT:    smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    fmov s7, s21
+; CHECK-SD-NEXT:    saddl2 v16.4s, v2.8h, v1.8h
 ; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
 ; CHECK-SD-NEXT:    saddl2 v2.4s, v4.8h, v3.8h
 ; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v0.4h
-; CHECK-SD-NEXT:    saddw v7.4s, v16.4s, v5.4h
-; CHECK-SD-NEXT:    saddw2 v0.4s, v6.4s, v0.8h
+; CHECK-SD-NEXT:    saddw v4.4s, v5.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v5.4s, v7.4s, v6.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v16.4s, v0.8h
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v5.8h
-; CHECK-SD-NEXT:    add v3.4s, v3.4s, v7.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v6.8h
+; CHECK-SD-NEXT:    add v3.4s, v3.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
@@ -3827,117 +3815,115 @@ define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %
 ; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-SD-NEXT:    .cfi_offset w29, -16
-; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b0, [sp, #16]
+; CHECK-SD-NEXT:    add x8, sp, #24
 ; CHECK-SD-NEXT:    ldr b1, [sp, #80]
-; CHECK-SD-NEXT:    add x10, sp, #88
-; CHECK-SD-NEXT:    ldr b2, [sp, #16]
-; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    fmov s2, w0
+; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #88
 ; CHECK-SD-NEXT:    ldr b3, [sp, #480]
-; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #24
-; CHECK-SD-NEXT:    ldr b4, [sp, #352]
-; CHECK-SD-NEXT:    mov v0.b[1], w1
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x10]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    add x10, sp, #360
-; CHECK-SD-NEXT:    ldr b5, [sp, #416]
-; CHECK-SD-NEXT:    add x8, sp, #104
-; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #32
-; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    add x11, sp, #424
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x10]
-; CHECK-SD-NEXT:    mov v0.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT:    add x9, sp, #368
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ldr b5, [sp, #352]
+; CHECK-SD-NEXT:    mov v2.b[1], w1
+; CHECK-SD-NEXT:    ldr b4, [sp, #416]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #96
+; CHECK-SD-NEXT:    add x13, sp, #128
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #40
-; CHECK-SD-NEXT:    add x12, sp, #496
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]
-; CHECK-SD-NEXT:    add x8, sp, #432
-; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
-; CHECK-SD-NEXT:    add x13, sp, #48
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v0.b[3], w3
-; CHECK-SD-NEXT:    add x10, sp, #112
-; CHECK-SD-NEXT:    add x8, sp, #504
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #376
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #440
-; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT:    add x11, sp, #120
-; CHECK-SD-NEXT:    add x8, sp, #56
-; CHECK-SD-NEXT:    mov v0.b[4], w4
-; CHECK-SD-NEXT:    add x13, sp, #512
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #384
-; CHECK-SD-NEXT:    add x11, sp, #448
-; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT:    add x12, sp, #128
-; CHECK-SD-NEXT:    add x10, sp, #64
-; CHECK-SD-NEXT:    add x8, sp, #520
-; CHECK-SD-NEXT:    mov v0.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #392
-; CHECK-SD-NEXT:    add x11, sp, #456
+; CHECK-SD-NEXT:    add x9, sp, #64
 ; CHECK-SD-NEXT:    ldr b6, [sp, #144]
 ; CHECK-SD-NEXT:    ldr b7, [sp, #544]
-; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-SD-NEXT:    mov v2.b[2], w2
+; CHECK-SD-NEXT:    add x8, sp, #72
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #112
 ; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT:    mov v0.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-SD-NEXT:    add x8, sp, #528
-; CHECK-SD-NEXT:    add x9, sp, #400
-; CHECK-SD-NEXT:    add x10, sp, #464
 ; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]
-; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT:    add x14, sp, #72
-; CHECK-SD-NEXT:    mov v0.b[7], w7
-; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT:    add x8, sp, #536
-; CHECK-SD-NEXT:    add x9, sp, #408
-; CHECK-SD-NEXT:    add x10, sp, #472
-; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x14]
-; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x10]
-; CHECK-SD-NEXT:    mov v16.s[0], v6.s[0]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #488
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #360
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
+; CHECK-SD-NEXT:    mov v2.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #496
+; CHECK-SD-NEXT:    add x12, sp, #120
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #368
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #432
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT:    mov v2.b[4], w4
+; CHECK-SD-NEXT:    add x11, sp, #440
+; CHECK-SD-NEXT:    add x10, sp, #504
+; CHECK-SD-NEXT:    add x12, sp, #376
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #512
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #448
+; CHECK-SD-NEXT:    add x13, sp, #384
+; CHECK-SD-NEXT:    mov v2.b[5], w5
+; CHECK-SD-NEXT:    add x10, sp, #136
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    add x10, sp, #392
+; CHECK-SD-NEXT:    add x11, sp, #456
+; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x10]
+; CHECK-SD-NEXT:    mov v2.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #464
+; CHECK-SD-NEXT:    add x10, sp, #528
+; CHECK-SD-NEXT:    add x11, sp, #400
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]
+; CHECK-SD-NEXT:    fmov s6, s6
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT:    add x10, sp, #536
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    mov v2.b[7], w7
+; CHECK-SD-NEXT:    add x9, sp, #472
+; CHECK-SD-NEXT:    add x11, sp, #408
+; CHECK-SD-NEXT:    fmov s7, s7
 ; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT:    mov v17.s[0], v7.s[0]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x11]
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
 ; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
 ; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT:    saddl v7.4s, v0.4h, v1.4h
-; CHECK-SD-NEXT:    saddl2 v0.4s, v0.8h, v1.8h
-; CHECK-SD-NEXT:    saddw v6.4s, v16.4s, v2.4h
-; CHECK-SD-NEXT:    saddl v1.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddl2 v3.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v5.4h
-; CHECK-SD-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-SD-NEXT:    add v6.4s, v7.4s, v6.4s
-; CHECK-SD-NEXT:    saddw2 v2.4s, v3.4s, v5.8h
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT:    add v0.4s, v6.4s, v0.4s
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT:    saddl2 v16.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    saddl2 v2.4s, v5.8h, v3.8h
+; CHECK-SD-NEXT:    saddl v3.4s, v5.4h, v3.4h
+; CHECK-SD-NEXT:    saddw v5.4s, v6.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v6.4s, v7.4s, v4.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v16.4s, v0.8h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v4.8h
+; CHECK-SD-NEXT:    add v3.4s, v3.4s, v6.4s
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -4457,7 +4443,6 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr b0, [x0, #32]
 ; CHECK-SD-NEXT:    ldr b1, [x1, #32]
-; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q3, q4, [x1]
 ; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    ldp q1, q2, [x0]
@@ -4466,15 +4451,15 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    umull2 v2.8h, v4.16b, v2.16b
 ; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-SD-NEXT:    umull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    uaddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    uaddl2 v3.4s, v1.8h, v2.8h
 ; CHECK-SD-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT:    uaddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT:    uaddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
 ; CHECK-SD-NEXT:    add w0, w8, w2
@@ -4903,23 +4888,22 @@ entry:
 define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v33i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr b1, [x0, #32]
-; CHECK-SD-NEXT:    ldp q3, q2, [x0]
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ldr b0, [x0, #32]
+; CHECK-SD-NEXT:    ldp q2, q1, [x0]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v3.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT:    ushll v5.8h, v3.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.8h, v1.16b, #0
 ; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-SD-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT:    uaddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
-; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
-; CHECK-SD-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v6.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    uaddl2 v5.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    uaddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
 ; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v4.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v3.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -5054,7 +5038,6 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr b0, [x0, #32]
 ; CHECK-SD-NEXT:    ldr b1, [x1, #32]
-; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q3, q4, [x1]
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    ldp q1, q2, [x0]
@@ -5063,15 +5046,15 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    smull2 v2.8h, v4.16b, v2.16b
 ; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
 ; CHECK-SD-NEXT:    smull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT:    saddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT:    saddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    saddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    saddl2 v3.4s, v1.8h, v2.8h
 ; CHECK-SD-NEXT:    saddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT:    saddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT:    saddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
 ; CHECK-SD-NEXT:    add w0, w8, w2
@@ -5506,268 +5489,266 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
 ; CHECK-SD-NEXT:    ldr b0, [sp, #344]
 ; CHECK-SD-NEXT:    add x8, sp, #352
 ; CHECK-SD-NEXT:    ldr b1, [sp, #80]
-; CHECK-SD-NEXT:    ldr b2, [sp, #216]
-; CHECK-SD-NEXT:    add x9, sp, #96
-; CHECK-SD-NEXT:    add x10, sp, #104
-; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #88
+; CHECK-SD-NEXT:    add x9, sp, #88
+; CHECK-SD-NEXT:    ldr b3, [sp, #216]
 ; CHECK-SD-NEXT:    ldr b4, [sp, #408]
-; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #360
-; CHECK-SD-NEXT:    add x12, sp, #248
-; CHECK-SD-NEXT:    add x13, sp, #432
-; CHECK-SD-NEXT:    add x11, sp, #384
-; CHECK-SD-NEXT:    ldr b5, [sp, #144]
+; CHECK-SD-NEXT:    add x9, sp, #224
+; CHECK-SD-NEXT:    add x12, sp, #416
+; CHECK-SD-NEXT:    add x10, sp, #376
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #224
-; CHECK-SD-NEXT:    ldr b6, [sp, #280]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #368
 ; CHECK-SD-NEXT:    add x9, sp, #232
-; CHECK-SD-NEXT:    ldr b16, [sp, #744]
-; CHECK-SD-NEXT:    ldr b17, [sp, #480]
+; CHECK-SD-NEXT:    add x12, sp, #112
+; CHECK-SD-NEXT:    ldr b6, [sp, #744]
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #104
+; CHECK-SD-NEXT:    add x11, sp, #384
 ; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #376
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #392
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    ldr b5, [sp, #144]
+; CHECK-SD-NEXT:    ldr b7, [sp, #480]
 ; CHECK-SD-NEXT:    ldr b18, [sp, #936]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x10]
-; CHECK-SD-NEXT:    add x9, sp, #240
-; CHECK-SD-NEXT:    add x10, sp, #392
 ; CHECK-SD-NEXT:    ldr b19, [sp, #672]
-; CHECK-SD-NEXT:    ldr b7, [sp, #16]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #112
-; CHECK-SD-NEXT:    ldr b21, [sp, #1000]
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #416
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #120
-; CHECK-SD-NEXT:    add x9, sp, #400
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #128
-; CHECK-SD-NEXT:    ldr b22, [sp, #736]
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    ldr b20, [sp, #808]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #240
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
 ; CHECK-SD-NEXT:    add x12, sp, #424
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #120
 ; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #152
-; CHECK-SD-NEXT:    add x8, sp, #136
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #440
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #288
-; CHECK-SD-NEXT:    add x10, sp, #256
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #296
+; CHECK-SD-NEXT:    add x12, sp, #752
+; CHECK-SD-NEXT:    ldr b22, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #248
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #432
+; CHECK-SD-NEXT:    add x12, sp, #760
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #256
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #128
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #440
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #768
+; CHECK-SD-NEXT:    add x10, sp, #264
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #160
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]
-; CHECK-SD-NEXT:    add x10, sp, #168
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #448
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #304
-; CHECK-SD-NEXT:    add x8, sp, #464
-; CHECK-SD-NEXT:    add x13, sp, #768
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #176
-; CHECK-SD-NEXT:    add x9, sp, #264
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #456
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #760
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #448
+; CHECK-SD-NEXT:    add x8, sp, #152
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #776
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #272
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #312
-; CHECK-SD-NEXT:    fmov s3, w0
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #320
-; CHECK-SD-NEXT:    add x12, sp, #680
-; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #184
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #776
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #752
-; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x8]
-; CHECK-SD-NEXT:    add x10, sp, #24
-; CHECK-SD-NEXT:    smull v22.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #496
-; CHECK-SD-NEXT:    mov v3.b[1], w1
-; CHECK-SD-NEXT:    add x9, sp, #192
-; CHECK-SD-NEXT:    ldr b20, [sp, #472]
-; CHECK-SD-NEXT:    ldr b23, [sp, #208]
-; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #944
-; CHECK-SD-NEXT:    add x9, sp, #328
-; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #688
+; CHECK-SD-NEXT:    add x8, sp, #488
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #456
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    add x9, sp, #784
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #496
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #464
+; CHECK-SD-NEXT:    add x10, sp, #944
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #792
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #552
+; CHECK-SD-NEXT:    add x11, sp, #952
 ; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #504
-; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #952
-; CHECK-SD-NEXT:    add x13, sp, #784
-; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #32
-; CHECK-SD-NEXT:    add x9, sp, #40
-; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #696
-; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #960
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x12]
-; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x11]
-; CHECK-SD-NEXT:    add x10, sp, #512
-; CHECK-SD-NEXT:    add x11, sp, #704
-; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x13]
-; CHECK-SD-NEXT:    add x12, sp, #792
-; CHECK-SD-NEXT:    sshll v24.4s, v22.4h, #0
-; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #968
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x11]
-; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #520
-; CHECK-SD-NEXT:    add x11, sp, #976
-; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #712
-; CHECK-SD-NEXT:    smull v20.8h, v23.8b, v20.8b
-; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #720
-; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #528
-; CHECK-SD-NEXT:    add x10, sp, #800
-; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #536
-; CHECK-SD-NEXT:    ldr b22, [sp, #872]
-; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #504
+; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #800
+; CHECK-SD-NEXT:    add x10, sp, #688
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #560
+; CHECK-SD-NEXT:    add x12, sp, #960
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #512
+; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #680
+; CHECK-SD-NEXT:    add x11, sp, #968
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #816
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    add x12, sp, #976
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #696
+; CHECK-SD-NEXT:    fmov s2, w0
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #824
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #568
+; CHECK-SD-NEXT:    add x11, sp, #576
+; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #704
+; CHECK-SD-NEXT:    mov v2.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #832
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #528
+; CHECK-SD-NEXT:    add x12, sp, #992
+; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #712
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #840
 ; CHECK-SD-NEXT:    add x11, sp, #984
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #584
 ; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #992
-; CHECK-SD-NEXT:    add x12, sp, #728
-; CHECK-SD-NEXT:    ldr b23, [sp, #608]
-; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #880
-; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x12]
-; CHECK-SD-NEXT:    add x10, sp, #816
-; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #552
-; CHECK-SD-NEXT:    add x12, sp, #616
-; CHECK-SD-NEXT:    mov v3.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #720
+; CHECK-SD-NEXT:    add x11, sp, #592
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #848
+; CHECK-SD-NEXT:    ldr b16, [sp, #472]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #728
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #856
+; CHECK-SD-NEXT:    ldr b17, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #600
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
 ; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
-; CHECK-SD-NEXT:    add x12, sp, #560
-; CHECK-SD-NEXT:    add x9, sp, #888
-; CHECK-SD-NEXT:    smull v17.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT:    ldr b18, [sp, #808]
-; CHECK-SD-NEXT:    ldr b19, [sp, #544]
-; CHECK-SD-NEXT:    add x13, sp, #624
-; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #864
+; CHECK-SD-NEXT:    smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT:    ldr b23, [sp, #736]
+; CHECK-SD-NEXT:    add x10, sp, #880
+; CHECK-SD-NEXT:    add x11, sp, #616
+; CHECK-SD-NEXT:    mov v2.b[2], w2
+; CHECK-SD-NEXT:    add x8, sp, #176
+; CHECK-SD-NEXT:    smull v7.8h, v19.8b, v18.8b
+; CHECK-SD-NEXT:    ldr b18, [sp, #1000]
+; CHECK-SD-NEXT:    ldr b19, [sp, #872]
+; CHECK-SD-NEXT:    smull v17.8h, v22.8b, v20.8b
+; CHECK-SD-NEXT:    ldr b20, [sp, #608]
+; CHECK-SD-NEXT:    ldr b21, [sp, #280]
+; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v18.8b
+; CHECK-SD-NEXT:    ldr b18, [sp, #16]
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #288
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #624
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #888
+; CHECK-SD-NEXT:    mov v2.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #296
+; CHECK-SD-NEXT:    add x9, sp, #32
+; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #632
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #896
-; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #824
-; CHECK-SD-NEXT:    add x10, sp, #48
-; CHECK-SD-NEXT:    ld1 { v23.b }[2], [x13]
-; CHECK-SD-NEXT:    mov v3.b[3], w3
-; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #832
-; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #568
-; CHECK-SD-NEXT:    add x12, sp, #632
-; CHECK-SD-NEXT:    add x9, sp, #904
-; CHECK-SD-NEXT:    add x13, sp, #640
-; CHECK-SD-NEXT:    ld1 { v23.b }[3], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #576
-; CHECK-SD-NEXT:    mov v3.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #840
-; CHECK-SD-NEXT:    add x10, sp, #56
-; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #184
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #304
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    mov v2.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #904
+; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #48
+; CHECK-SD-NEXT:    add x11, sp, #320
+; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #912
-; CHECK-SD-NEXT:    ld1 { v23.b }[4], [x13]
-; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #848
-; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #584
-; CHECK-SD-NEXT:    add x12, sp, #648
-; CHECK-SD-NEXT:    mov v3.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #592
-; CHECK-SD-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x10, sp, #648
+; CHECK-SD-NEXT:    mov v2.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    add x8, sp, #192
+; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
 ; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #856
-; CHECK-SD-NEXT:    add x9, sp, #920
-; CHECK-SD-NEXT:    add x13, sp, #656
-; CHECK-SD-NEXT:    add x10, sp, #64
-; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[6], [x13]
-; CHECK-SD-NEXT:    mov v3.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #864
-; CHECK-SD-NEXT:    add x11, sp, #600
-; CHECK-SD-NEXT:    add x9, sp, #928
-; CHECK-SD-NEXT:    add x12, sp, #664
-; CHECK-SD-NEXT:    mov v21.s[0], v24.s[0]
-; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v23.b }[7], [x12]
+; CHECK-SD-NEXT:    add x10, sp, #920
+; CHECK-SD-NEXT:    add x11, sp, #656
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT:    sshll v22.4s, v22.4h, #0
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
+; CHECK-SD-NEXT:    mov v2.b[6], w6
+; CHECK-SD-NEXT:    add x9, sp, #328
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #64
+; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #928
+; CHECK-SD-NEXT:    add x9, sp, #664
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT:    fmov s22, s22
+; CHECK-SD-NEXT:    mov v2.b[7], w7
 ; CHECK-SD-NEXT:    add x8, sp, #200
-; CHECK-SD-NEXT:    mov v3.b[7], w7
-; CHECK-SD-NEXT:    add x10, sp, #336
+; CHECK-SD-NEXT:    add x9, sp, #336
+; CHECK-SD-NEXT:    add x10, sp, #72
 ; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #72
-; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x10]
-; CHECK-SD-NEXT:    smull v18.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
-; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v22.8b
-; CHECK-SD-NEXT:    sshll v20.4s, v20.4h, #0
+; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
+; CHECK-SD-NEXT:    smull v19.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    saddw v1.4s, v21.4s, v16.4h
-; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT:    saddw v1.4s, v22.4s, v6.4h
+; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
 ; CHECK-SD-NEXT:    smull v3.8h, v5.8b, v4.8b
-; CHECK-SD-NEXT:    smull v4.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT:    mov v19.s[0], v20.s[0]
-; CHECK-SD-NEXT:    saddl2 v5.4s, v18.8h, v17.8h
-; CHECK-SD-NEXT:    saddl v7.4s, v18.4h, v17.4h
-; CHECK-SD-NEXT:    saddl2 v6.4s, v16.8h, v22.8h
-; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v22.4h
-; CHECK-SD-NEXT:    saddl2 v17.4s, v2.8h, v0.8h
-; CHECK-SD-NEXT:    saddl2 v16.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddw v2.4s, v19.4s, v2.4h
-; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT:    smull v4.8h, v18.8b, v21.8b
+; CHECK-SD-NEXT:    fmov s5, s16
+; CHECK-SD-NEXT:    saddl2 v16.4s, v17.8h, v7.8h
+; CHECK-SD-NEXT:    saddl2 v6.4s, v6.8h, v19.8h
+; CHECK-SD-NEXT:    saddl v7.4s, v17.4h, v7.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v19.4h
+; CHECK-SD-NEXT:    saddl2 v18.4s, v2.8h, v0.8h
+; CHECK-SD-NEXT:    saddl2 v17.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    add v5.4s, v6.4s, v16.4s
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v7.4s
-; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT:    add v6.4s, v18.4s, v17.4s
 ; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
@@ -6577,146 +6558,144 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
 ; CHECK-SD-NEXT:    ldr b0, [sp, #80]
 ; CHECK-SD-NEXT:    add x8, sp, #88
 ; CHECK-SD-NEXT:    ldr b2, [sp, #144]
-; CHECK-SD-NEXT:    add x9, sp, #152
+; CHECK-SD-NEXT:    add x9, sp, #96
 ; CHECK-SD-NEXT:    ldr b3, [sp, #16]
-; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    add x11, sp, #112
 ; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #24
-; CHECK-SD-NEXT:    add x8, sp, #96
-; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
-; CHECK-SD-NEXT:    ldr b5, [sp, #480]
-; CHECK-SD-NEXT:    fmov s1, w0
-; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x8, sp, #152
 ; CHECK-SD-NEXT:    add x12, sp, #168
-; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
 ; CHECK-SD-NEXT:    ldr b4, [sp, #608]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    ldr b5, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT:    add x13, sp, #184
+; CHECK-SD-NEXT:    ldr b6, [sp, #672]
+; CHECK-SD-NEXT:    mov v1.b[1], w1
+; CHECK-SD-NEXT:    ldr b7, [sp, #544]
+; CHECK-SD-NEXT:    add x10, sp, #120
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #32
-; CHECK-SD-NEXT:    add x13, sp, #496
+; CHECK-SD-NEXT:    add x9, sp, #128
 ; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v1.b[1], w1
-; CHECK-SD-NEXT:    ldr b6, [sp, #672]
-; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    add x9, sp, #120
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #40
 ; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
-; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
-; CHECK-SD-NEXT:    add x12, sp, #616
-; CHECK-SD-NEXT:    ldr b16, [sp, #544]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #48
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
 ; CHECK-SD-NEXT:    add x12, sp, #176
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #680
-; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT:    add x14, sp, #56
+; CHECK-SD-NEXT:    add x15, sp, #560
+; CHECK-SD-NEXT:    ldr b17, [sp, #736]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #40
+; CHECK-SD-NEXT:    mov v1.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #616
 ; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    add x11, sp, #192
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-SD-NEXT:    ldr b16, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #624
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #488
+; CHECK-SD-NEXT:    add x13, sp, #680
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #56
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
-; CHECK-SD-NEXT:    mov v1.b[2], w2
-; CHECK-SD-NEXT:    add x8, sp, #128
-; CHECK-SD-NEXT:    add x14, sp, #184
-; CHECK-SD-NEXT:    add x11, sp, #136
-; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
 ; CHECK-SD-NEXT:    add x13, sp, #552
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x14]
-; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x13]
-; CHECK-SD-NEXT:    add x14, sp, #624
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #688
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #496
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x14]
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    mov v1.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
 ; CHECK-SD-NEXT:    add x13, sp, #504
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x14]
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #560
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x15]
+; CHECK-SD-NEXT:    add x12, sp, #64
+; CHECK-SD-NEXT:    add x14, sp, #632
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x14]
 ; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v1.b[3], w3
-; CHECK-SD-NEXT:    add x9, sp, #64
-; CHECK-SD-NEXT:    add x15, sp, #632
-; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x15]
-; CHECK-SD-NEXT:    add x8, sp, #696
-; CHECK-SD-NEXT:    add x9, sp, #568
-; CHECK-SD-NEXT:    add x11, sp, #512
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT:    add x8, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #696
+; CHECK-SD-NEXT:    add x11, sp, #568
+; CHECK-SD-NEXT:    add x12, sp, #512
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #640
 ; CHECK-SD-NEXT:    mov v1.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #704
-; CHECK-SD-NEXT:    add x9, sp, #576
-; CHECK-SD-NEXT:    add x11, sp, #520
-; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    ldr b18, [sp, #736]
-; CHECK-SD-NEXT:    add x12, sp, #192
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x12]
-; CHECK-SD-NEXT:    add x8, sp, #648
-; CHECK-SD-NEXT:    add x9, sp, #528
-; CHECK-SD-NEXT:    add x11, sp, #712
-; CHECK-SD-NEXT:    add x12, sp, #584
-; CHECK-SD-NEXT:    sshll v18.8h, v18.8b, #0
+; CHECK-SD-NEXT:    add x11, sp, #576
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #704
+; CHECK-SD-NEXT:    add x12, sp, #520
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #648
+; CHECK-SD-NEXT:    add x11, sp, #528
+; CHECK-SD-NEXT:    add x13, sp, #584
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #712
 ; CHECK-SD-NEXT:    mov v1.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x12]
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT:    add x8, sp, #656
-; CHECK-SD-NEXT:    add x9, sp, #536
-; CHECK-SD-NEXT:    add x11, sp, #720
-; CHECK-SD-NEXT:    add x12, sp, #592
-; CHECK-SD-NEXT:    sshll v18.4s, v18.4h, #0
-; CHECK-SD-NEXT:    ldr b7, [sp, #208]
-; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #656
+; CHECK-SD-NEXT:    add x12, sp, #720
+; CHECK-SD-NEXT:    add x13, sp, #592
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #536
+; CHECK-SD-NEXT:    sshll v17.4s, v17.4h, #0
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
 ; CHECK-SD-NEXT:    mov v1.b[6], w6
-; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT:    add x8, sp, #664
-; CHECK-SD-NEXT:    add x9, sp, #728
-; CHECK-SD-NEXT:    add x11, sp, #600
-; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT:    add x9, sp, #664
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #728
+; CHECK-SD-NEXT:    add x12, sp, #600
+; CHECK-SD-NEXT:    fmov s17, s17
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #136
 ; CHECK-SD-NEXT:    add x10, sp, #200
+; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
 ; CHECK-SD-NEXT:    mov v1.b[7], w7
-; CHECK-SD-NEXT:    add x9, sp, #72
-; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    add x11, sp, #72
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]
 ; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x11]
 ; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
 ; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
 ; CHECK-SD-NEXT:    saddw v17.4s, v17.4s, v5.4h
+; CHECK-SD-NEXT:    fmov s16, s16
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    mov v18.s[0], v7.s[0]
 ; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT:    saddl2 v7.4s, v16.8h, v6.8h
+; CHECK-SD-NEXT:    saddl2 v18.4s, v7.8h, v6.8h
 ; CHECK-SD-NEXT:    saddl2 v5.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT:    saddl v6.4s, v16.4h, v6.4h
+; CHECK-SD-NEXT:    saddl v6.4s, v7.4h, v6.4h
 ; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v4.4h
 ; CHECK-SD-NEXT:    saddl2 v17.4s, v1.8h, v0.8h
-; CHECK-SD-NEXT:    saddl2 v16.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT:    saddw v1.4s, v18.4s, v1.4h
-; CHECK-SD-NEXT:    add v5.4s, v5.4s, v7.4s
-; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT:    saddl2 v7.4s, v3.8h, v2.8h
+; CHECK-SD-NEXT:    saddw v1.4s, v16.4s, v1.4h
 ; CHECK-SD-NEXT:    saddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    add v5.4s, v5.4s, v18.4s
+; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT:    add v6.4s, v17.4s, v7.4s
 ; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    add v1.4s, v4.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index dab16a39802df..6c55abdfef791 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -5,7 +5,7 @@
 ; RUN: llc < %s -mtriple=aarch64 -O2 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,GI,GI-NOFP16
 
 ; The canonical form uses <i32 0, i32 poison, i32 poison, ... > for the zero vector operand
-; We use equivalent zeroinitializer instead which has no functional change
+; We use equivalent zeroinitializer for testing instead which has no functional change
 
 define <2 x double> @testv2f64(<2 x double> noundef %vec) {
 ; SD-LABEL: testv2f64:
@@ -26,9 +26,7 @@ entry:
 define <4 x float> @testv4f32(<4 x float> noundef %vec) {
 ; SD-LABEL: testv4f32:
 ; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.s[0], v0.s[0]
-; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    fmov s0, s0
 ; SD-NEXT:    ret
 ;
 ; GI-LABEL: testv4f32:
@@ -64,12 +62,17 @@ entry:
 
 
 define <8 x half> @testv8f16(<8 x half> noundef %vec) {
-; SD-LABEL: testv8f16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8f16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8f16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8f16:
 ; GI:       // %bb.0: // %entry
@@ -86,13 +89,18 @@ entry:
 
 
 define <4 x half> @testv4f16(<4 x half> noundef %vec) {
-; SD-LABEL: testv4f16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4f16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4f16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4f16:
 ; GI:       // %bb.0: // %entry
@@ -109,12 +117,17 @@ entry:
 
 
 define <8 x bfloat> @testv8bf16(<8 x bfloat> noundef %vec) {
-; SD-LABEL: testv8bf16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8bf16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8bf16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8bf16:
 ; GI:       // %bb.0: // %entry
@@ -131,13 +144,18 @@ entry:
 
 
 define <4 x bfloat> @testv4bf16(<4 x bfloat> noundef %vec) {
-; SD-LABEL: testv4bf16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4bf16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4bf16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4bf16:
 ; GI:       // %bb.0: // %entry
@@ -172,9 +190,7 @@ entry:
 define <4 x i32> @testv4i32(<4 x i32> noundef %vec) {
 ; SD-LABEL: testv4i32:
 ; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.s[0], v0.s[0]
-; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    fmov s0, s0
 ; SD-NEXT:    ret
 ;
 ; GI-LABEL: testv4i32:
@@ -210,12 +226,17 @@ entry:
 
 
 define <8 x i16> @testv8i16(<8 x i16> noundef %vec) {
-; SD-LABEL: testv8i16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8i16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8i16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8i16:
 ; GI:       // %bb.0: // %entry
@@ -230,13 +251,18 @@ entry:
 
 
 define <4 x i16> @testv4i16(<4 x i16> noundef %vec) {
-; SD-LABEL: testv4i16:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4i16:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4i16:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4i16:
 ; GI:       // %bb.0: // %entry
@@ -300,5 +326,3 @@ entry:
 ; CHECK: {{.*}}
 ; GI-FP16: {{.*}}
 ; GI-NOFP16: {{.*}}
-; SD-FP16: {{.*}}
-; SD-NOFP16: {{.*}}



More information about the llvm-branch-commits mailing list