[llvm-branch-commits] [llvm] [AArch64][DAG] Improve codegen for clearing upper vector lanes (PR #237259)

Kieran B via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 2 01:52:07 PST 2026


https://github.com/kieroxide updated https://github.com/llvm/llvm-project/pull/237259

>From 75cf59c1e561716f79c466bec530cc3c1fa320fe Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:15 +0000
Subject: [PATCH 1/2] [AArch64][DAG] Improve codegen for clearing upper vector
 lanes

Add patterns that use scalar fmov instruction to clear the upper lanes of a vector
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  41 ++
 .../CodeGen/AArch64/shuffle-insert-zero.ll    | 497 +++++++++---------
 2 files changed, 293 insertions(+), 245 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 75739d8cf83ef4..14e0d72190398e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8753,6 +8753,47 @@ defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v4i16, scalar_to_vector, INSvi8lane
 defm : Neon_INS_elt_ext_pattern<v16i8, v8i8, v2i32, scalar_to_vector_v2f32, INSvi8lane, DUPi8, bsub, VecIndex_x4>;
 defm : Neon_INS_elt_ext_pattern<v8i16, v4i16, v2i32, scalar_to_vector_v2f32, INSvi16lane, DUPi16, hsub, VecIndex_x2>;
 
+multiclass ExtractInsertLane0IntoZeroVector<ValueType VecVT, ValueType ElemTy> {
+  let Predicates = [HasFullFP16] in {
+    def : Pat<(VecVT (vector_insert immAllZerosV,
+                (ElemTy (vector_extract (VecVT V128:$Rn), (i64 0))), (i64 0))),
+              (VecVT (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (VecVT V128:$Rn), hsub))), hsub))>;
+  }
+}
+
+defm : ExtractInsertLane0IntoZeroVector<v8i16,  i32>;
+defm : ExtractInsertLane0IntoZeroVector<v8f16,  f16>;
+defm : ExtractInsertLane0IntoZeroVector<v8bf16, bf16>;
+
+let Predicates = [HasFullFP16] in {
+  def : Pat<(v4i16 (vector_insert immAllZerosV, (i32 (vector_extract 
+                (v8i16 (insert_subvector (v8i16 undef), (v4i16 V64:$Rn), (i64 0))), 
+                (i64 0))), 
+              (i64 0))),
+            (v4i16 (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (v4i16 V64:$Rn), hsub))), hsub))>;
+    
+  def : Pat<(v4f16 (vector_insert immAllZerosV, 
+              (f16 (vector_extract 
+                (v8f16 (insert_subvector (v8f16 undef), (v4f16 V64:$Rn), (i64 0))), 
+                (i64 0))), 
+              (i64 0))),
+            (v4f16 (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (v4f16 V64:$Rn), hsub))), hsub))>;
+        
+  def : Pat<(v4bf16 (vector_insert immAllZerosV, 
+              (bf16 (vector_extract 
+                (v8bf16 (insert_subvector (v8bf16 undef), (v4bf16 V64:$Rn), (i64 0))), 
+                (i64 0))), 
+              (i64 0))),
+            (v4bf16 (SUBREG_TO_REG (FMOVHr (f16 (EXTRACT_SUBREG (v4bf16 V64:$Rn), hsub))), hsub))>;
+}
+
+// extract from lane 0 and insert into lane 0 of a zero vector
+// is equivalent to clearing upper bits of a register, we can use fmov
+def : Pat<(v4i32 (vector_insert immAllZerosV, (i32 (vector_extract (v4i32 V128:$Rn), (i64 0))), (i64 0))),
+          (v4i32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4i32 V128:$Rn), ssub)), ssub))>;
+def : Pat<(v4f32 (vector_insert immAllZerosV, (f32 (vector_extract (v4f32 V128:$Rn), (i64 0))), (i64 0))),
+          (v4f32 (SUBREG_TO_REG (FMOVSr (EXTRACT_SUBREG (v4f32 V128:$Rn), ssub)), ssub))>;
+
 // bitcast of an extract
 // f32 bitcast(vector_extract(v4i32 src, 0)) -> EXTRACT_SUBREG(src)
 def : Pat<(f32 (bitconvert (i32 (vector_extract v16i8:$src, (i64 0))))),
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index 3b4b137b023050..edef4a56bcd4e5 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -1,36 +1,37 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,SD,SD-FP16
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,SD,SD-NOFP16
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,GI,GI-FP16
+; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,GI,GI-NOFP16
 
 define <2 x double> @testv2f64_0(<2 x double> noundef %vec) {
-; CHECK-SD-LABEL: testv2f64_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2f64_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f64_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    mov v1.d[1], v1.d[0]
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f64_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    mov v1.d[1], v1.d[0]
+; GI-NEXT:    mov v0.d[1], v1.d[1]
+; GI-NEXT:    ret
 entry:
   %vecinit1 = shufflevector <2 x double> %vec, <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
   ret <2 x double> %vecinit1
 }
 
 define <2 x double> @testv2f64_1(<2 x double> noundef %vec) {
-; CHECK-SD-LABEL: testv2f64_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2f64_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f64_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f64_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <2 x double> <double 0.000000e+00, double poison>, <2 x double> %vec, <2 x i32> <i32 2, i32 0>
   ret <2 x double> %vecins
@@ -49,59 +50,59 @@ entry:
 }
 
 define <4 x float> @testv4f32_1(<4 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv4f32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4f32_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.s[0], v0.s[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4f32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4f32_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    dup v1.4s, v1.s[0]
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, <4 x float> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
   ret <4 x float> %vecins
 }
 
 define <2 x float> @testv2f32_0(<2 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv2f32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2f32_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.s[1], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f32_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    mov v1.s[1], v1.s[0]
+; GI-NEXT:    mov v0.s[1], v1.s[1]
+; GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <2 x float> %vec, <2 x float> <float poison, float 0.000000e+00>, <2 x i32> <i32 0, i32 3>
   ret <2 x float> %vecinit3
 }
 
 define <2 x float> @testv2f32_1(<2 x float> noundef %vec) {
-; CHECK-SD-LABEL: testv2f32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2f32_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2f32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2f32_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <2 x float> <float 0.000000e+00, float poison>, <2 x float> %vec, <2 x i32> <i32 2, i32 0>
   ret <2 x float> %vecins
@@ -120,64 +121,64 @@ entry:
 }
 
 define <8 x half> @testv8f16_1(<8 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv8f16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv8f16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv8f16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI7_0
-; CHECK-GI-NEXT:    movi d31, #0000000000000000
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv8f16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
+; GI-NEXT:    adrp x8, .LCPI7_0
+; GI-NEXT:    movi d31, #0000000000000000
+; GI-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
+; GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <8 x half> <half 0.000000e+00, half poison, half poison, half poison, half poison, half poison, half poison, half poison>, <8 x half> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x half> %vecins
 }
 
 define <4 x half> @testv4f16_0(<4 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv4f16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4f16_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi d1, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4f16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4f16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <4 x half> %vec, <4 x half> <half poison, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x half> %vecinit3
 }
 
 define <4 x half> @testv4f16_1(<4 x half> noundef %vec) {
-; CHECK-SD-LABEL: testv4f16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4f16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi d1, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4f16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4f16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <4 x half> <half 0.000000e+00, half poison, half poison, half poison>, <4 x half> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
   ret <4 x half> %vecins
@@ -196,64 +197,64 @@ entry:
 }
 
 define <8 x bfloat> @testv8bf16_1(<8 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv8bf16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv8bf16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv8bf16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI11_0
-; CHECK-GI-NEXT:    movi d31, #0000000000000000
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI11_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv8bf16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    // kill: def $q0 killed $q0 def $q31_q0
+; GI-NEXT:    adrp x8, .LCPI11_0
+; GI-NEXT:    movi d31, #0000000000000000
+; GI-NEXT:    ldr q1, [x8, :lo12:.LCPI11_0]
+; GI-NEXT:    tbl v0.16b, { v31.16b, v0.16b }, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <8 x bfloat> <bfloat 0.000000e+00, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison, bfloat poison>, <8 x bfloat> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x bfloat> %vecins
 }
 
 define <4 x bfloat> @testv4bf16_0(<4 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv4bf16_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4bf16_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi d1, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4bf16_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4bf16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <4 x bfloat> %vec, <4 x bfloat> <bfloat poison, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x bfloat> %vecinit3
 }
 
 define <4 x bfloat> @testv4bf16_1(<4 x bfloat> noundef %vec) {
-; CHECK-SD-LABEL: testv4bf16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4bf16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi d1, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4bf16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4bf16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <4 x bfloat> <bfloat 0.000000e+00, bfloat poison, bfloat poison, bfloat poison>, <4 x bfloat> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
   ret <4 x bfloat> %vecins
@@ -261,32 +262,32 @@ entry:
 
 
 define <2 x i64> @testv2i64_0(<2 x i64> noundef %vec) {
-; CHECK-SD-LABEL: testv2i64_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2i64_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i64_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[1]
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i64_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    mov v1.d[1], xzr
+; GI-NEXT:    mov v0.d[1], v1.d[1]
+; GI-NEXT:    ret
 entry:
   %vecinit1 = shufflevector <2 x i64> %vec, <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
   ret <2 x i64> %vecinit1
 }
 
 define <2 x i64> @testv2i64_1(<2 x i64> noundef %vec) {
-; CHECK-SD-LABEL: testv2i64_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v0.d[1], xzr
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2i64_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    mov v0.d[1], xzr
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i64_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i64_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    zip1 v0.2d, v0.2d, v1.2d
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <2 x i64> <i64 0, i64 poison>, <2 x i64> %vec, <2 x i32> <i32 2, i32 0>
   ret <2 x i64> %vecins
@@ -305,58 +306,58 @@ entry:
 }
 
 define <4 x i32> @testv4i32_1(<4 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv4i32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv4i32_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.s[0], v0.s[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4i32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    dup v1.4s, v1.s[0]
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv4i32_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    dup v1.4s, v1.s[0]
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>, <4 x i32> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
   ret <4 x i32> %vecins
 }
 
 define <2 x i32> @testv2i32_0(<2 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv2i32_0:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2i32_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i32_0:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[1], wzr
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i32_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    mov v1.s[1], wzr
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    mov v0.s[1], v1.s[1]
+; GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <2 x i32> %vec, <2 x i32> <i32 poison, i32 0>, <2 x i32> <i32 0, i32 3>
   ret <2 x i32> %vecinit3
 }
 
 define <2 x i32> @testv2i32_1(<2 x i32> noundef %vec) {
-; CHECK-SD-LABEL: testv2i32_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], wzr
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv2i32_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v0.s[1], wzr
+; SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv2i32_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv2i32_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    zip1 v0.2s, v0.2s, v1.2s
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <2 x i32> <i32 0, i32 poison>, <2 x i32> %vec, <2 x i32> <i32 2, i32 0>
   ret <2 x i32> %vecins
@@ -375,19 +376,19 @@ entry:
 }
 
 define <8 x i16> @testv8i16_1(<8 x i16> noundef %vec) {
-; CHECK-SD-LABEL: testv8i16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv8i16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv8i16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    adrp x8, .LCPI21_0
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv8i16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    adrp x8, .LCPI21_0
+; GI-NEXT:    ldr q1, [x8, :lo12:.LCPI21_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <8 x i16> <i16 0, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>, <8 x i16> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i16> %vecins
@@ -407,22 +408,23 @@ entry:
 }
 
 define <4 x i16> @testv4i16_1(<4 x i16> noundef %vec) {
-; CHECK-SD-LABEL: testv4i16_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-SD-NEXT:    fmov d0, d1
-; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv4i16_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    dup v1.4h, v1.h[0]
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    ret
+; SD-LABEL: testv4i16_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    movi v1.2d, #0000000000000000
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NEXT:    fmov d0, d1
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv4i16_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    dup v1.4h, v1.h[0]
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <4 x i16> <i16 0, i16 poison, i16 poison, i16 poison>, <4 x i16> %vec, <4 x i32> <i32 4, i32 0, i32 0, i32 0>
   ret <4 x i16> %vecins
@@ -466,25 +468,30 @@ entry:
 }
 
 define <8 x i8> @testv8i8_1(<8 x i8> noundef %vec) {
-; CHECK-SD-LABEL: testv8i8_1:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    movi d1, #0xffffffffffffff00
-; CHECK-SD-NEXT:    mov v0.d[1], v0.d[0]
-; CHECK-SD-NEXT:    tbl v0.8b, { v0.16b }, v1.8b
-; CHECK-SD-NEXT:    ret
+; SD-LABEL: testv8i8_1:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NEXT:    movi d1, #0xffffffffffffff00
+; SD-NEXT:    mov v0.d[1], v0.d[0]
+; SD-NEXT:    tbl v0.8b, { v0.16b }, v1.8b
+; SD-NEXT:    ret
 ;
-; CHECK-GI-LABEL: testv8i8_1:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi d1, #0000000000000000
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    adrp x8, .LCPI27_0
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI27_0]
-; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; GI-LABEL: testv8i8_1:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi d1, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    adrp x8, .LCPI27_0
+; GI-NEXT:    mov v0.d[1], v1.d[0]
+; GI-NEXT:    ldr d1, [x8, :lo12:.LCPI27_0]
+; GI-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; GI-NEXT:    ret
 entry:
   %vecins = shufflevector <8 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <8 x i8> %vec, <8 x i32> <i32 8, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <8 x i8> %vecins
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GI-FP16: {{.*}}
+; GI-NOFP16: {{.*}}
+; SD-FP16: {{.*}}
+; SD-NOFP16: {{.*}}

>From 6aeb284137ea6ea6dba49f572da7f2e4204a190c Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Mon, 2 Nov 2026 09:40:32 +0000
Subject: [PATCH 2/2] Update Test Checks

---
 .../aarch64-neon-vector-insert-uaddlv.ll      |   30 +-
 .../CodeGen/AArch64/build-vector-extract.ll   |    4 +-
 llvm/test/CodeGen/AArch64/neon-dotreduce.ll   | 1289 ++++++++---------
 .../CodeGen/AArch64/shuffle-insert-zero.ll    |  289 ++--
 4 files changed, 839 insertions(+), 773 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
index 91eda8d5523975..ad6fc44e1e7faa 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll
@@ -30,8 +30,8 @@ define void @insert_vec_v4i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v4i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    uaddlv.8h s1, v0
-; CHECK-NEXT:    mov.s v0[0], v1[0]
+; CHECK-NEXT:    uaddlv.8h s0, v0
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ucvtf.4s v0, v0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
@@ -48,11 +48,10 @@ define void @insert_vec_v16i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.8h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
 
@@ -68,14 +67,13 @@ define void @insert_vec_v23i32_uaddlv_from_v8i16(ptr %0) {
 ; CHECK-LABEL: insert_vec_v23i32_uaddlv_from_v8i16:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    str wzr, [x0, #88]
 ; CHECK-NEXT:    str xzr, [x0, #80]
 ; CHECK-NEXT:    uaddlv.8h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #16]
 ; CHECK-NEXT:    stp q0, q0, [x0, #48]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    str q1, [x0]
 ; CHECK-NEXT:    ret
 
@@ -254,10 +252,9 @@ define void @insert_vec_v16i64_uaddlv_from_v4i16(ptr %0) {
 ; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.4h s1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    fmov x8, d2
-; CHECK-NEXT:    mov.d x9, v2[1]
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    fmov x8, d1
+; CHECK-NEXT:    mov.d x9, v1[1]
 ; CHECK-NEXT:    ucvtf s1, x8
 ; CHECK-NEXT:    ucvtf s3, x9
 ; CHECK-NEXT:    mov.s v2[0], v1[0]
@@ -346,8 +343,8 @@ define void @insert_vec_v8i32_uaddlv_from_v4i32(ptr %0) {
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
 ; CHECK-NEXT:    stp xzr, xzr, [x0, #16]
-; CHECK-NEXT:    uaddlv.4s d1, v0
-; CHECK-NEXT:    mov.s v0[0], v1[0]
+; CHECK-NEXT:    uaddlv.4s d0, v0
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ucvtf.4s v0, v0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
@@ -365,11 +362,10 @@ define void @insert_vec_v16i32_uaddlv_from_v4i32(ptr %0) {
 ; CHECK-LABEL: insert_vec_v16i32_uaddlv_from_v4i32:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-NEXT:    uaddlv.4s d1, v0
 ; CHECK-NEXT:    stp q0, q0, [x0, #32]
-; CHECK-NEXT:    mov.s v2[0], v1[0]
-; CHECK-NEXT:    ucvtf.4s v1, v2
+; CHECK-NEXT:    fmov s1, s1
+; CHECK-NEXT:    ucvtf.4s v1, v1
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/AArch64/build-vector-extract.ll b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
index f3e40021b616a2..0cfbc939c12a82 100644
--- a/llvm/test/CodeGen/AArch64/build-vector-extract.ll
+++ b/llvm/test/CodeGen/AArch64/build-vector-extract.ll
@@ -15,9 +15,7 @@ define <2 x i64> @extract0_i32_zext_insert0_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
 ; CHECK-LABEL: extract0_i32_zext_insert0_i64_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    fmov s0, s0
 ; CHECK-NEXT:    ret
   %e = extractelement <4 x i32> %x, i32 0
   %z = zext i32 %e to i64
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 2483b476799f61..0fb3cb2643d651 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -321,9 +321,8 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
 ; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -406,10 +405,9 @@ define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v5i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr d0, [x0]
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -447,9 +445,8 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
 ; CHECK-SD-NEXT:    ldr d0, [x0]
 ; CHECK-SD-NEXT:    ldr d1, [x1]
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
 ; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -533,14 +530,12 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
 ; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
-; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0
-; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT:    mov v1.s[0], v5.s[0]
-; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT:    sshll2 v1.4s, v2.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-SD-NEXT:    fmov s1, s1
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v2.4h
+; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -682,14 +677,12 @@ define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
 ; CHECK-SD-NEXT:    sshll v1.8h, v2.8b, #0
-; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[0], v5.s[0]
-; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h
-; CHECK-SD-NEXT:    saddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT:    sshll2 v3.4s, v1.8h, #0
+; CHECK-SD-NEXT:    fmov s2, s2
+; CHECK-SD-NEXT:    fmov s3, s3
+; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v3.4s, v1.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -2310,7 +2303,6 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-LABEL: test_udot_v25i8:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldp q3, q0, [x1]
-; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q2, q1, [x0]
 ; CHECK-SD-NEXT:    umull2 v4.8h, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b
@@ -2319,10 +2311,10 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    ushll v3.4s, v4.4h, #0
 ; CHECK-SD-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h
 ; CHECK-SD-NEXT:    uaddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uaddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    uaddw v2.4s, v3.4s, v2.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -2637,20 +2629,19 @@ entry:
 define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v25i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q2, q1, [x0]
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-SD-NEXT:    uaddl v1.4s, v4.4h, v1.4h
-; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
-; CHECK-SD-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
-; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    ldp q1, q0, [x0]
+; CHECK-SD-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-SD-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v4.4s, v3.8h, v0.8h
+; CHECK-SD-NEXT:    uaddl v0.4s, v3.4h, v0.4h
+; CHECK-SD-NEXT:    fmov s2, s2
+; CHECK-SD-NEXT:    uaddw2 v3.4s, v4.4s, v1.8h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
+; CHECK-SD-NEXT:    uaddw v1.4s, v2.4s, v1.4h
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ret
@@ -2739,7 +2730,6 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-LABEL: test_sdot_v25i8:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldp q3, q0, [x1]
-; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q2, q1, [x0]
 ; CHECK-SD-NEXT:    smull2 v4.8h, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b
@@ -2748,10 +2738,10 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    sshll v3.4s, v4.4h, #0
 ; CHECK-SD-NEXT:    saddl2 v4.4s, v1.8h, v0.8h
 ; CHECK-SD-NEXT:    saddl v0.4s, v1.4h, v0.4h
-; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]
+; CHECK-SD-NEXT:    fmov s3, s3
 ; CHECK-SD-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    saddw v2.4s, v3.4s, v2.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
@@ -3073,203 +3063,201 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25
 ; CHECK-SD-NEXT:    add x8, sp, #224
 ; CHECK-SD-NEXT:    ldr b1, [sp, #16]
 ; CHECK-SD-NEXT:    ldr b2, [sp, #280]
-; CHECK-SD-NEXT:    add x9, sp, #240
+; CHECK-SD-NEXT:    add x9, sp, #288
 ; CHECK-SD-NEXT:    ldr b4, [sp, #80]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #24
-; CHECK-SD-NEXT:    add x10, sp, #48
+; CHECK-SD-NEXT:    ldr b5, [sp, #152]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #232
-; CHECK-SD-NEXT:    add x11, sp, #96
-; CHECK-SD-NEXT:    ldr b5, [sp, #152]
-; CHECK-SD-NEXT:    add x12, sp, #168
-; CHECK-SD-NEXT:    ldr b6, [sp, #616]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #40
+; CHECK-SD-NEXT:    add x10, sp, #296
+; CHECK-SD-NEXT:    add x11, sp, #64
 ; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #32
-; CHECK-SD-NEXT:    fmov s3, w0
+; CHECK-SD-NEXT:    ldr b6, [sp, #616]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #288
+; CHECK-SD-NEXT:    add x8, sp, #240
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #96
+; CHECK-SD-NEXT:    fmov s3, w0
 ; CHECK-SD-NEXT:    ldr b7, [sp, #416]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #40
-; CHECK-SD-NEXT:    ldr b22, [sp, #744]
-; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #248
-; CHECK-SD-NEXT:    mov v3.b[1], w1
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #88
-; CHECK-SD-NEXT:    ldr b23, [sp, #544]
+; CHECK-SD-NEXT:    ldr b16, [sp, #344]
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x9]
 ; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #256
-; CHECK-SD-NEXT:    ldr b19, [sp, #680]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #296
-; CHECK-SD-NEXT:    ldr b20, [sp, #480]
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    add x10, sp, #160
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #304
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #56
-; CHECK-SD-NEXT:    add x10, sp, #264
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #64
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #248
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    mov v3.b[1], w1
+; CHECK-SD-NEXT:    ldr b17, [sp, #144]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #48
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #256
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    add x8, sp, #264
+; CHECK-SD-NEXT:    ldr b20, [sp, #744]
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #272
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #72
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    ldr b21, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #304
 ; CHECK-SD-NEXT:    mov v3.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #72
+; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ldr b17, [sp, #680]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #104
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #312
 ; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #112
-; CHECK-SD-NEXT:    add x10, sp, #128
+; CHECK-SD-NEXT:    add x8, sp, #320
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x11]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #320
-; CHECK-SD-NEXT:    ldr b21, [sp, #552]
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #176
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #624
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #120
-; CHECK-SD-NEXT:    add x9, sp, #328
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #184
-; CHECK-SD-NEXT:    add x11, sp, #192
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #632
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #624
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x11, sp, #176
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x10]
+; CHECK-SD-NEXT:    add x9, sp, #632
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x11]
+; CHECK-SD-NEXT:    add x8, sp, #328
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #120
+; CHECK-SD-NEXT:    add x11, sp, #184
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #128
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
 ; CHECK-SD-NEXT:    add x8, sp, #336
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #192
+; CHECK-SD-NEXT:    ldr b18, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #200
+; CHECK-SD-NEXT:    ldr b19, [sp, #552]
 ; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #656
-; CHECK-SD-NEXT:    smull v23.8h, v23.8b, v22.8b
+; CHECK-SD-NEXT:    add x8, sp, #648
 ; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #648
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #200
-; CHECK-SD-NEXT:    add x10, sp, #136
-; CHECK-SD-NEXT:    ldr b22, [sp, #352]
-; CHECK-SD-NEXT:    add x12, sp, #360
-; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #424
+; CHECK-SD-NEXT:    smull v21.8h, v21.8b, v20.8b
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #656
+; CHECK-SD-NEXT:    ldr b20, [sp, #352]
 ; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #208
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    add x9, sp, #136
 ; CHECK-SD-NEXT:    add x10, sp, #488
-; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #560
-; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #440
-; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x11]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #432
-; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #208
+; CHECK-SD-NEXT:    add x11, sp, #560
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #440
+; CHECK-SD-NEXT:    add x12, sp, #360
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    mov v3.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #696
 ; CHECK-SD-NEXT:    add x11, sp, #496
 ; CHECK-SD-NEXT:    add x12, sp, #568
 ; CHECK-SD-NEXT:    add x13, sp, #368
-; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x11]
-; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x13]
 ; CHECK-SD-NEXT:    add x10, sp, #448
+; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #704
 ; CHECK-SD-NEXT:    mov v3.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #504
 ; CHECK-SD-NEXT:    add x11, sp, #576
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #696
 ; CHECK-SD-NEXT:    add x12, sp, #376
-; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
+; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x12]
+; CHECK-SD-NEXT:    add x9, sp, #712
 ; CHECK-SD-NEXT:    add x11, sp, #512
-; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #504
 ; CHECK-SD-NEXT:    add x12, sp, #584
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #704
-; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
 ; CHECK-SD-NEXT:    add x13, sp, #384
 ; CHECK-SD-NEXT:    mov v3.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x13]
 ; CHECK-SD-NEXT:    add x10, sp, #456
-; CHECK-SD-NEXT:    ldr b16, [sp, #344]
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #712
-; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x11]
-; CHECK-SD-NEXT:    ldr b17, [sp, #144]
+; CHECK-SD-NEXT:    add x9, sp, #720
 ; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
 ; CHECK-SD-NEXT:    add x10, sp, #520
 ; CHECK-SD-NEXT:    add x11, sp, #592
 ; CHECK-SD-NEXT:    add x12, sp, #392
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x12]
 ; CHECK-SD-NEXT:    mov v3.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #720
-; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x12]
-; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
+; CHECK-SD-NEXT:    add x9, sp, #728
 ; CHECK-SD-NEXT:    add x8, sp, #664
 ; CHECK-SD-NEXT:    add x10, sp, #464
 ; CHECK-SD-NEXT:    add x11, sp, #528
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #728
 ; CHECK-SD-NEXT:    add x12, sp, #600
+; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x9]
 ; CHECK-SD-NEXT:    add x13, sp, #400
 ; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x13]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #736
 ; CHECK-SD-NEXT:    mov v3.b[7], w7
-; CHECK-SD-NEXT:    sshll v18.4s, v16.4h, #0
-; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
 ; CHECK-SD-NEXT:    add x8, sp, #672
 ; CHECK-SD-NEXT:    add x10, sp, #472
-; CHECK-SD-NEXT:    add x11, sp, #608
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    add x11, sp, #608
 ; CHECK-SD-NEXT:    add x12, sp, #408
-; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x12]
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x12]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
 ; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]
-; CHECK-SD-NEXT:    sshll v23.4s, v23.4h, #0
+; CHECK-SD-NEXT:    sshll v21.4s, v21.4h, #0
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    smull v1.8h, v4.8b, v2.8b
 ; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v5.8b
-; CHECK-SD-NEXT:    smull v3.8h, v20.8b, v19.8b
-; CHECK-SD-NEXT:    smull v4.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT:    smull v5.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT:    mov v16.s[0], v23.s[0]
-; CHECK-SD-NEXT:    saddl2 v6.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    smull v3.8h, v18.8b, v17.8b
+; CHECK-SD-NEXT:    fmov s5, s16
+; CHECK-SD-NEXT:    smull v4.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT:    smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    fmov s7, s21
+; CHECK-SD-NEXT:    saddl2 v16.4s, v2.8h, v1.8h
 ; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
 ; CHECK-SD-NEXT:    saddl2 v2.4s, v4.8h, v3.8h
 ; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v0.4h
-; CHECK-SD-NEXT:    saddw v7.4s, v16.4s, v5.4h
-; CHECK-SD-NEXT:    saddw2 v0.4s, v6.4s, v0.8h
+; CHECK-SD-NEXT:    saddw v4.4s, v5.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v5.4s, v7.4s, v6.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v16.4s, v0.8h
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v5.8h
-; CHECK-SD-NEXT:    add v3.4s, v3.4s, v7.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v6.8h
+; CHECK-SD-NEXT:    add v3.4s, v3.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
@@ -3827,117 +3815,115 @@ define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %
 ; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-SD-NEXT:    .cfi_offset w29, -16
-; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    ldr b0, [sp, #16]
+; CHECK-SD-NEXT:    add x8, sp, #24
 ; CHECK-SD-NEXT:    ldr b1, [sp, #80]
-; CHECK-SD-NEXT:    add x10, sp, #88
-; CHECK-SD-NEXT:    ldr b2, [sp, #16]
-; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    fmov s2, w0
+; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #88
 ; CHECK-SD-NEXT:    ldr b3, [sp, #480]
-; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #24
-; CHECK-SD-NEXT:    ldr b4, [sp, #352]
-; CHECK-SD-NEXT:    mov v0.b[1], w1
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x10]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    add x10, sp, #360
-; CHECK-SD-NEXT:    ldr b5, [sp, #416]
-; CHECK-SD-NEXT:    add x8, sp, #104
-; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #32
-; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    add x11, sp, #424
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x10]
-; CHECK-SD-NEXT:    mov v0.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT:    add x9, sp, #368
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ldr b5, [sp, #352]
+; CHECK-SD-NEXT:    mov v2.b[1], w1
+; CHECK-SD-NEXT:    ldr b4, [sp, #416]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #96
+; CHECK-SD-NEXT:    add x13, sp, #128
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #40
-; CHECK-SD-NEXT:    add x12, sp, #496
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]
-; CHECK-SD-NEXT:    add x8, sp, #432
-; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]
-; CHECK-SD-NEXT:    add x13, sp, #48
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v0.b[3], w3
-; CHECK-SD-NEXT:    add x10, sp, #112
-; CHECK-SD-NEXT:    add x8, sp, #504
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #376
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #440
-; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT:    add x11, sp, #120
-; CHECK-SD-NEXT:    add x8, sp, #56
-; CHECK-SD-NEXT:    mov v0.b[4], w4
-; CHECK-SD-NEXT:    add x13, sp, #512
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #384
-; CHECK-SD-NEXT:    add x11, sp, #448
-; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT:    add x12, sp, #128
-; CHECK-SD-NEXT:    add x10, sp, #64
-; CHECK-SD-NEXT:    add x8, sp, #520
-; CHECK-SD-NEXT:    mov v0.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #392
-; CHECK-SD-NEXT:    add x11, sp, #456
+; CHECK-SD-NEXT:    add x9, sp, #64
 ; CHECK-SD-NEXT:    ldr b6, [sp, #144]
 ; CHECK-SD-NEXT:    ldr b7, [sp, #544]
-; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-SD-NEXT:    mov v2.b[2], w2
+; CHECK-SD-NEXT:    add x8, sp, #72
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #112
 ; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT:    mov v0.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-SD-NEXT:    add x8, sp, #528
-; CHECK-SD-NEXT:    add x9, sp, #400
-; CHECK-SD-NEXT:    add x10, sp, #464
 ; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]
-; CHECK-SD-NEXT:    movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT:    add x14, sp, #72
-; CHECK-SD-NEXT:    mov v0.b[7], w7
-; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT:    add x8, sp, #536
-; CHECK-SD-NEXT:    add x9, sp, #408
-; CHECK-SD-NEXT:    add x10, sp, #472
-; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x14]
-; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x10]
-; CHECK-SD-NEXT:    mov v16.s[0], v6.s[0]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #488
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #360
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #424
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
+; CHECK-SD-NEXT:    mov v2.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #496
+; CHECK-SD-NEXT:    add x12, sp, #120
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #368
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #432
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT:    mov v2.b[4], w4
+; CHECK-SD-NEXT:    add x11, sp, #440
+; CHECK-SD-NEXT:    add x10, sp, #504
+; CHECK-SD-NEXT:    add x12, sp, #376
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #512
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #448
+; CHECK-SD-NEXT:    add x13, sp, #384
+; CHECK-SD-NEXT:    mov v2.b[5], w5
+; CHECK-SD-NEXT:    add x10, sp, #136
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    add x10, sp, #392
+; CHECK-SD-NEXT:    add x11, sp, #456
+; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x10]
+; CHECK-SD-NEXT:    mov v2.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #464
+; CHECK-SD-NEXT:    add x10, sp, #528
+; CHECK-SD-NEXT:    add x11, sp, #400
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]
+; CHECK-SD-NEXT:    fmov s6, s6
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT:    add x10, sp, #536
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    mov v2.b[7], w7
+; CHECK-SD-NEXT:    add x9, sp, #472
+; CHECK-SD-NEXT:    add x11, sp, #408
+; CHECK-SD-NEXT:    fmov s7, s7
 ; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-SD-NEXT:    mov v17.s[0], v7.s[0]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x11]
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
 ; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
 ; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT:    saddl v7.4s, v0.4h, v1.4h
-; CHECK-SD-NEXT:    saddl2 v0.4s, v0.8h, v1.8h
-; CHECK-SD-NEXT:    saddw v6.4s, v16.4s, v2.4h
-; CHECK-SD-NEXT:    saddl v1.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddl2 v3.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v5.4h
-; CHECK-SD-NEXT:    saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-SD-NEXT:    add v6.4s, v7.4s, v6.4s
-; CHECK-SD-NEXT:    saddw2 v2.4s, v3.4s, v5.8h
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-SD-NEXT:    add v0.4s, v6.4s, v0.4s
-; CHECK-SD-NEXT:    add v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-SD-NEXT:    saddl2 v16.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    saddl2 v2.4s, v5.8h, v3.8h
+; CHECK-SD-NEXT:    saddl v3.4s, v5.4h, v3.4h
+; CHECK-SD-NEXT:    saddw v5.4s, v6.4s, v0.4h
+; CHECK-SD-NEXT:    saddw v6.4s, v7.4s, v4.4h
+; CHECK-SD-NEXT:    saddw2 v0.4s, v16.4s, v0.8h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v4.8h
+; CHECK-SD-NEXT:    add v3.4s, v3.4s, v6.4s
+; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -4457,7 +4443,6 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr b0, [x0, #32]
 ; CHECK-SD-NEXT:    ldr b1, [x1, #32]
-; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q3, q4, [x1]
 ; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    ldp q1, q2, [x0]
@@ -4466,15 +4451,15 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    umull2 v2.8h, v4.16b, v2.16b
 ; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-SD-NEXT:    umull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    uaddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    uaddl2 v3.4s, v1.8h, v2.8h
 ; CHECK-SD-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT:    uaddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT:    uaddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
 ; CHECK-SD-NEXT:    add w0, w8, w2
@@ -4903,23 +4888,22 @@ entry:
 define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-SD-LABEL: test_udot_v33i8_nomla:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldr b1, [x0, #32]
-; CHECK-SD-NEXT:    ldp q3, q2, [x0]
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ldr b0, [x0, #32]
+; CHECK-SD-NEXT:    ldp q2, q1, [x0]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v3.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #0
-; CHECK-SD-NEXT:    ushll v5.8h, v3.8b, #0
+; CHECK-SD-NEXT:    ushll2 v1.8h, v1.16b, #0
 ; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-SD-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-SD-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT:    uaddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s
-; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h
-; CHECK-SD-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    uaddl2 v6.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    uaddl2 v5.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    uaddl v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
 ; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v4.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v3.4h
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
@@ -5054,7 +5038,6 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    ldr b0, [x0, #32]
 ; CHECK-SD-NEXT:    ldr b1, [x1, #32]
-; CHECK-SD-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-SD-NEXT:    ldp q3, q4, [x1]
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
 ; CHECK-SD-NEXT:    ldp q1, q2, [x0]
@@ -5063,15 +5046,15 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
 ; CHECK-SD-NEXT:    smull2 v2.8h, v4.16b, v2.16b
 ; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
 ; CHECK-SD-NEXT:    smull2 v1.8h, v3.16b, v1.16b
-; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]
-; CHECK-SD-NEXT:    saddl2 v3.4s, v6.8h, v5.8h
-; CHECK-SD-NEXT:    saddl2 v0.4s, v1.8h, v2.8h
+; CHECK-SD-NEXT:    fmov s0, s0
+; CHECK-SD-NEXT:    saddl2 v4.4s, v6.8h, v5.8h
+; CHECK-SD-NEXT:    saddl2 v3.4s, v1.8h, v2.8h
 ; CHECK-SD-NEXT:    saddl v1.4s, v1.4h, v2.4h
-; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s
-; CHECK-SD-NEXT:    saddw v2.4s, v7.4s, v6.4h
-; CHECK-SD-NEXT:    saddw v2.4s, v2.4s, v5.4h
-; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    add v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v6.4h
+; CHECK-SD-NEXT:    saddw v0.4s, v0.4s, v5.4h
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w8, s0
 ; CHECK-SD-NEXT:    add w0, w8, w2
@@ -5506,268 +5489,266 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
 ; CHECK-SD-NEXT:    ldr b0, [sp, #344]
 ; CHECK-SD-NEXT:    add x8, sp, #352
 ; CHECK-SD-NEXT:    ldr b1, [sp, #80]
-; CHECK-SD-NEXT:    ldr b2, [sp, #216]
-; CHECK-SD-NEXT:    add x9, sp, #96
-; CHECK-SD-NEXT:    add x10, sp, #104
-; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #88
+; CHECK-SD-NEXT:    add x9, sp, #88
+; CHECK-SD-NEXT:    ldr b3, [sp, #216]
 ; CHECK-SD-NEXT:    ldr b4, [sp, #408]
-; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #360
-; CHECK-SD-NEXT:    add x12, sp, #248
-; CHECK-SD-NEXT:    add x13, sp, #432
-; CHECK-SD-NEXT:    add x11, sp, #384
-; CHECK-SD-NEXT:    ldr b5, [sp, #144]
+; CHECK-SD-NEXT:    add x9, sp, #224
+; CHECK-SD-NEXT:    add x12, sp, #416
+; CHECK-SD-NEXT:    add x10, sp, #376
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #96
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #224
-; CHECK-SD-NEXT:    ldr b6, [sp, #280]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
 ; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #368
 ; CHECK-SD-NEXT:    add x9, sp, #232
-; CHECK-SD-NEXT:    ldr b16, [sp, #744]
-; CHECK-SD-NEXT:    ldr b17, [sp, #480]
+; CHECK-SD-NEXT:    add x12, sp, #112
+; CHECK-SD-NEXT:    ldr b6, [sp, #744]
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #104
+; CHECK-SD-NEXT:    add x11, sp, #384
 ; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #376
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #392
+; CHECK-SD-NEXT:    add x9, sp, #400
+; CHECK-SD-NEXT:    ldr b5, [sp, #144]
+; CHECK-SD-NEXT:    ldr b7, [sp, #480]
 ; CHECK-SD-NEXT:    ldr b18, [sp, #936]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x10]
-; CHECK-SD-NEXT:    add x9, sp, #240
-; CHECK-SD-NEXT:    add x10, sp, #392
 ; CHECK-SD-NEXT:    ldr b19, [sp, #672]
-; CHECK-SD-NEXT:    ldr b7, [sp, #16]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #112
-; CHECK-SD-NEXT:    ldr b21, [sp, #1000]
-; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #416
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #120
-; CHECK-SD-NEXT:    add x9, sp, #400
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #128
-; CHECK-SD-NEXT:    ldr b22, [sp, #736]
-; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    ldr b20, [sp, #808]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #240
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]
 ; CHECK-SD-NEXT:    add x12, sp, #424
-; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #120
 ; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #152
-; CHECK-SD-NEXT:    add x8, sp, #136
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #440
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #288
-; CHECK-SD-NEXT:    add x10, sp, #256
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #296
+; CHECK-SD-NEXT:    add x12, sp, #752
+; CHECK-SD-NEXT:    ldr b22, [sp, #544]
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #248
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #432
+; CHECK-SD-NEXT:    add x12, sp, #760
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #256
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #128
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x11]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #440
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #768
+; CHECK-SD-NEXT:    add x10, sp, #264
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
 ; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #160
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]
-; CHECK-SD-NEXT:    add x10, sp, #168
-; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #448
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #304
-; CHECK-SD-NEXT:    add x8, sp, #464
-; CHECK-SD-NEXT:    add x13, sp, #768
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #176
-; CHECK-SD-NEXT:    add x9, sp, #264
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #456
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #760
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #136
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #448
+; CHECK-SD-NEXT:    add x8, sp, #152
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #776
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #272
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #312
-; CHECK-SD-NEXT:    fmov s3, w0
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #320
-; CHECK-SD-NEXT:    add x12, sp, #680
-; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #184
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #776
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #752
-; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x8]
-; CHECK-SD-NEXT:    add x10, sp, #24
-; CHECK-SD-NEXT:    smull v22.8h, v22.8b, v21.8b
-; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #496
-; CHECK-SD-NEXT:    mov v3.b[1], w1
-; CHECK-SD-NEXT:    add x9, sp, #192
-; CHECK-SD-NEXT:    ldr b20, [sp, #472]
-; CHECK-SD-NEXT:    ldr b23, [sp, #208]
-; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #944
-; CHECK-SD-NEXT:    add x9, sp, #328
-; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #688
+; CHECK-SD-NEXT:    add x8, sp, #488
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #456
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    add x9, sp, #784
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #496
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #464
+; CHECK-SD-NEXT:    add x10, sp, #944
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #792
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #552
+; CHECK-SD-NEXT:    add x11, sp, #952
 ; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #504
-; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #952
-; CHECK-SD-NEXT:    add x13, sp, #784
-; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #32
-; CHECK-SD-NEXT:    add x9, sp, #40
-; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #696
-; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #960
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x12]
-; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x11]
-; CHECK-SD-NEXT:    add x10, sp, #512
-; CHECK-SD-NEXT:    add x11, sp, #704
-; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x13]
-; CHECK-SD-NEXT:    add x12, sp, #792
-; CHECK-SD-NEXT:    sshll v24.4s, v22.4h, #0
-; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #968
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x11]
-; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #520
-; CHECK-SD-NEXT:    add x11, sp, #976
-; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #712
-; CHECK-SD-NEXT:    smull v20.8h, v23.8b, v20.8b
-; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #720
-; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #528
-; CHECK-SD-NEXT:    add x10, sp, #800
-; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #536
-; CHECK-SD-NEXT:    ldr b22, [sp, #872]
-; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #504
+; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #800
+; CHECK-SD-NEXT:    add x10, sp, #688
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #560
+; CHECK-SD-NEXT:    add x12, sp, #960
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #512
+; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #680
+; CHECK-SD-NEXT:    add x11, sp, #968
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #816
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #520
+; CHECK-SD-NEXT:    add x12, sp, #976
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #696
+; CHECK-SD-NEXT:    fmov s2, w0
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #824
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #568
+; CHECK-SD-NEXT:    add x11, sp, #576
+; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #704
+; CHECK-SD-NEXT:    mov v2.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #832
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #528
+; CHECK-SD-NEXT:    add x12, sp, #992
+; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #712
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #840
 ; CHECK-SD-NEXT:    add x11, sp, #984
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #584
 ; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #992
-; CHECK-SD-NEXT:    add x12, sp, #728
-; CHECK-SD-NEXT:    ldr b23, [sp, #608]
-; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #880
-; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x12]
-; CHECK-SD-NEXT:    add x10, sp, #816
-; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #552
-; CHECK-SD-NEXT:    add x12, sp, #616
-; CHECK-SD-NEXT:    mov v3.b[2], w2
-; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[1], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #720
+; CHECK-SD-NEXT:    add x11, sp, #592
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #848
+; CHECK-SD-NEXT:    ldr b16, [sp, #472]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #536
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #728
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #856
+; CHECK-SD-NEXT:    ldr b17, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #600
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x10]
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
 ; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b
-; CHECK-SD-NEXT:    add x12, sp, #560
-; CHECK-SD-NEXT:    add x9, sp, #888
-; CHECK-SD-NEXT:    smull v17.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT:    ldr b18, [sp, #808]
-; CHECK-SD-NEXT:    ldr b19, [sp, #544]
-; CHECK-SD-NEXT:    add x13, sp, #624
-; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #864
+; CHECK-SD-NEXT:    smull v6.8h, v7.8b, v6.8b
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT:    ldr b23, [sp, #736]
+; CHECK-SD-NEXT:    add x10, sp, #880
+; CHECK-SD-NEXT:    add x11, sp, #616
+; CHECK-SD-NEXT:    mov v2.b[2], w2
+; CHECK-SD-NEXT:    add x8, sp, #176
+; CHECK-SD-NEXT:    smull v7.8h, v19.8b, v18.8b
+; CHECK-SD-NEXT:    ldr b18, [sp, #1000]
+; CHECK-SD-NEXT:    ldr b19, [sp, #872]
+; CHECK-SD-NEXT:    smull v17.8h, v22.8b, v20.8b
+; CHECK-SD-NEXT:    ldr b20, [sp, #608]
+; CHECK-SD-NEXT:    ldr b21, [sp, #280]
+; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v18.8b
+; CHECK-SD-NEXT:    ldr b18, [sp, #16]
+; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x10]
+; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #288
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
+; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #624
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #888
+; CHECK-SD-NEXT:    mov v2.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x9]
+; CHECK-SD-NEXT:    add x10, sp, #296
+; CHECK-SD-NEXT:    add x9, sp, #32
+; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #632
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #896
-; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #824
-; CHECK-SD-NEXT:    add x10, sp, #48
-; CHECK-SD-NEXT:    ld1 { v23.b }[2], [x13]
-; CHECK-SD-NEXT:    mov v3.b[3], w3
-; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #832
-; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #568
-; CHECK-SD-NEXT:    add x12, sp, #632
-; CHECK-SD-NEXT:    add x9, sp, #904
-; CHECK-SD-NEXT:    add x13, sp, #640
-; CHECK-SD-NEXT:    ld1 { v23.b }[3], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #576
-; CHECK-SD-NEXT:    mov v3.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #840
-; CHECK-SD-NEXT:    add x10, sp, #56
-; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #184
+; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]
+; CHECK-SD-NEXT:    add x11, sp, #304
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    mov v2.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #904
+; CHECK-SD-NEXT:    add x9, sp, #640
+; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x8]
+; CHECK-SD-NEXT:    add x10, sp, #312
+; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #48
+; CHECK-SD-NEXT:    add x11, sp, #320
+; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x10]
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x9]
 ; CHECK-SD-NEXT:    add x9, sp, #912
-; CHECK-SD-NEXT:    ld1 { v23.b }[4], [x13]
-; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #848
-; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]
-; CHECK-SD-NEXT:    add x11, sp, #584
-; CHECK-SD-NEXT:    add x12, sp, #648
-; CHECK-SD-NEXT:    mov v3.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[5], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #592
-; CHECK-SD-NEXT:    movi v21.2d, #0000000000000000
+; CHECK-SD-NEXT:    add x10, sp, #648
+; CHECK-SD-NEXT:    mov v2.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]
+; CHECK-SD-NEXT:    add x10, sp, #56
+; CHECK-SD-NEXT:    add x8, sp, #192
+; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]
 ; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #856
-; CHECK-SD-NEXT:    add x9, sp, #920
-; CHECK-SD-NEXT:    add x13, sp, #656
-; CHECK-SD-NEXT:    add x10, sp, #64
-; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v23.b }[6], [x13]
-; CHECK-SD-NEXT:    mov v3.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #864
-; CHECK-SD-NEXT:    add x11, sp, #600
-; CHECK-SD-NEXT:    add x9, sp, #928
-; CHECK-SD-NEXT:    add x12, sp, #664
-; CHECK-SD-NEXT:    mov v21.s[0], v24.s[0]
-; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v23.b }[7], [x12]
+; CHECK-SD-NEXT:    add x10, sp, #920
+; CHECK-SD-NEXT:    add x11, sp, #656
+; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x10]
+; CHECK-SD-NEXT:    sshll v22.4s, v22.4h, #0
+; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]
+; CHECK-SD-NEXT:    mov v2.b[6], w6
+; CHECK-SD-NEXT:    add x9, sp, #328
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #64
+; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #928
+; CHECK-SD-NEXT:    add x9, sp, #664
+; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]
+; CHECK-SD-NEXT:    fmov s22, s22
+; CHECK-SD-NEXT:    mov v2.b[7], w7
 ; CHECK-SD-NEXT:    add x8, sp, #200
-; CHECK-SD-NEXT:    mov v3.b[7], w7
-; CHECK-SD-NEXT:    add x10, sp, #336
+; CHECK-SD-NEXT:    add x9, sp, #336
+; CHECK-SD-NEXT:    add x10, sp, #72
 ; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #72
-; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x10]
-; CHECK-SD-NEXT:    smull v18.8h, v19.8b, v18.8b
-; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
-; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
-; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v22.8b
-; CHECK-SD-NEXT:    sshll v20.4s, v20.4h, #0
+; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]
+; CHECK-SD-NEXT:    smull v19.8h, v20.8b, v19.8b
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
 ; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b
-; CHECK-SD-NEXT:    saddw v1.4s, v21.4s, v16.4h
-; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v2.8b
+; CHECK-SD-NEXT:    saddw v1.4s, v22.4s, v6.4h
+; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b
 ; CHECK-SD-NEXT:    smull v3.8h, v5.8b, v4.8b
-; CHECK-SD-NEXT:    smull v4.8h, v7.8b, v6.8b
-; CHECK-SD-NEXT:    mov v19.s[0], v20.s[0]
-; CHECK-SD-NEXT:    saddl2 v5.4s, v18.8h, v17.8h
-; CHECK-SD-NEXT:    saddl v7.4s, v18.4h, v17.4h
-; CHECK-SD-NEXT:    saddl2 v6.4s, v16.8h, v22.8h
-; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v22.4h
-; CHECK-SD-NEXT:    saddl2 v17.4s, v2.8h, v0.8h
-; CHECK-SD-NEXT:    saddl2 v16.4s, v4.8h, v3.8h
-; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
-; CHECK-SD-NEXT:    saddw v2.4s, v19.4s, v2.4h
-; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT:    smull v4.8h, v18.8b, v21.8b
+; CHECK-SD-NEXT:    fmov s5, s16
+; CHECK-SD-NEXT:    saddl2 v16.4s, v17.8h, v7.8h
+; CHECK-SD-NEXT:    saddl2 v6.4s, v6.8h, v19.8h
+; CHECK-SD-NEXT:    saddl v7.4s, v17.4h, v7.4h
+; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v19.4h
+; CHECK-SD-NEXT:    saddl2 v18.4s, v2.8h, v0.8h
+; CHECK-SD-NEXT:    saddl2 v17.4s, v4.8h, v3.8h
+; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h
+; CHECK-SD-NEXT:    add v5.4s, v6.4s, v16.4s
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v7.4s
-; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h
+; CHECK-SD-NEXT:    add v6.4s, v18.4s, v17.4s
 ; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h
 ; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s
@@ -6577,146 +6558,144 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
 ; CHECK-SD-NEXT:    ldr b0, [sp, #80]
 ; CHECK-SD-NEXT:    add x8, sp, #88
 ; CHECK-SD-NEXT:    ldr b2, [sp, #144]
-; CHECK-SD-NEXT:    add x9, sp, #152
+; CHECK-SD-NEXT:    add x9, sp, #96
 ; CHECK-SD-NEXT:    ldr b3, [sp, #16]
-; CHECK-SD-NEXT:    add x11, sp, #104
+; CHECK-SD-NEXT:    add x11, sp, #112
 ; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #24
-; CHECK-SD-NEXT:    add x8, sp, #96
-; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]
-; CHECK-SD-NEXT:    ldr b5, [sp, #480]
-; CHECK-SD-NEXT:    fmov s1, w0
-; CHECK-SD-NEXT:    add x10, sp, #112
+; CHECK-SD-NEXT:    add x8, sp, #152
 ; CHECK-SD-NEXT:    add x12, sp, #168
-; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #24
 ; CHECK-SD-NEXT:    ldr b4, [sp, #608]
-; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    fmov s1, w0
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #160
+; CHECK-SD-NEXT:    ldr b5, [sp, #480]
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-SD-NEXT:    add x13, sp, #184
+; CHECK-SD-NEXT:    ldr b6, [sp, #672]
+; CHECK-SD-NEXT:    mov v1.b[1], w1
+; CHECK-SD-NEXT:    ldr b7, [sp, #544]
+; CHECK-SD-NEXT:    add x10, sp, #120
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #32
-; CHECK-SD-NEXT:    add x13, sp, #496
+; CHECK-SD-NEXT:    add x9, sp, #128
 ; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v1.b[1], w1
-; CHECK-SD-NEXT:    ldr b6, [sp, #672]
-; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #488
-; CHECK-SD-NEXT:    add x9, sp, #120
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]
-; CHECK-SD-NEXT:    add x11, sp, #40
 ; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]
-; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
-; CHECK-SD-NEXT:    add x12, sp, #616
-; CHECK-SD-NEXT:    ldr b16, [sp, #544]
-; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #48
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]
 ; CHECK-SD-NEXT:    add x12, sp, #176
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #680
-; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT:    add x14, sp, #56
+; CHECK-SD-NEXT:    add x15, sp, #560
+; CHECK-SD-NEXT:    ldr b17, [sp, #736]
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #40
+; CHECK-SD-NEXT:    mov v1.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #616
 ; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT:    add x12, sp, #48
+; CHECK-SD-NEXT:    add x11, sp, #192
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-SD-NEXT:    ldr b16, [sp, #208]
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #624
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #488
+; CHECK-SD-NEXT:    add x13, sp, #680
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]
 ; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #56
-; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]
-; CHECK-SD-NEXT:    mov v1.b[2], w2
-; CHECK-SD-NEXT:    add x8, sp, #128
-; CHECK-SD-NEXT:    add x14, sp, #184
-; CHECK-SD-NEXT:    add x11, sp, #136
-; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]
 ; CHECK-SD-NEXT:    add x13, sp, #552
-; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x14]
-; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x13]
-; CHECK-SD-NEXT:    add x14, sp, #624
-; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #688
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT:    add x13, sp, #496
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x14]
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #688
+; CHECK-SD-NEXT:    mov v1.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
 ; CHECK-SD-NEXT:    add x13, sp, #504
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x14]
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #560
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x15]
+; CHECK-SD-NEXT:    add x12, sp, #64
+; CHECK-SD-NEXT:    add x14, sp, #632
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x11]
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x14]
 ; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]
-; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x8]
-; CHECK-SD-NEXT:    mov v1.b[3], w3
-; CHECK-SD-NEXT:    add x9, sp, #64
-; CHECK-SD-NEXT:    add x15, sp, #632
-; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x15]
-; CHECK-SD-NEXT:    add x8, sp, #696
-; CHECK-SD-NEXT:    add x9, sp, #568
-; CHECK-SD-NEXT:    add x11, sp, #512
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
-; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT:    add x8, sp, #640
+; CHECK-SD-NEXT:    add x9, sp, #696
+; CHECK-SD-NEXT:    add x11, sp, #568
+; CHECK-SD-NEXT:    add x12, sp, #512
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #640
 ; CHECK-SD-NEXT:    mov v1.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #704
-; CHECK-SD-NEXT:    add x9, sp, #576
-; CHECK-SD-NEXT:    add x11, sp, #520
-; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]
-; CHECK-SD-NEXT:    ldr b18, [sp, #736]
-; CHECK-SD-NEXT:    add x12, sp, #192
-; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x12]
-; CHECK-SD-NEXT:    add x8, sp, #648
-; CHECK-SD-NEXT:    add x9, sp, #528
-; CHECK-SD-NEXT:    add x11, sp, #712
-; CHECK-SD-NEXT:    add x12, sp, #584
-; CHECK-SD-NEXT:    sshll v18.8h, v18.8b, #0
+; CHECK-SD-NEXT:    add x11, sp, #576
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #704
+; CHECK-SD-NEXT:    add x12, sp, #520
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x11]
+; CHECK-SD-NEXT:    add x9, sp, #648
+; CHECK-SD-NEXT:    add x11, sp, #528
+; CHECK-SD-NEXT:    add x13, sp, #584
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x12]
+; CHECK-SD-NEXT:    add x12, sp, #712
 ; CHECK-SD-NEXT:    mov v1.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]
-; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x12]
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT:    movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT:    add x8, sp, #656
-; CHECK-SD-NEXT:    add x9, sp, #536
-; CHECK-SD-NEXT:    add x11, sp, #720
-; CHECK-SD-NEXT:    add x12, sp, #592
-; CHECK-SD-NEXT:    sshll v18.4s, v18.4h, #0
-; CHECK-SD-NEXT:    ldr b7, [sp, #208]
-; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x11]
-; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #656
+; CHECK-SD-NEXT:    add x12, sp, #720
+; CHECK-SD-NEXT:    add x13, sp, #592
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #536
+; CHECK-SD-NEXT:    sshll v17.4s, v17.4h, #0
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x12]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x13]
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]
 ; CHECK-SD-NEXT:    mov v1.b[6], w6
-; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-SD-NEXT:    add x8, sp, #664
-; CHECK-SD-NEXT:    add x9, sp, #728
-; CHECK-SD-NEXT:    add x11, sp, #600
-; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]
-; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]
-; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-SD-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT:    add x9, sp, #664
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x11]
+; CHECK-SD-NEXT:    add x11, sp, #728
+; CHECK-SD-NEXT:    add x12, sp, #600
+; CHECK-SD-NEXT:    fmov s17, s17
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x11]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x12]
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #136
 ; CHECK-SD-NEXT:    add x10, sp, #200
+; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #0
 ; CHECK-SD-NEXT:    mov v1.b[7], w7
-; CHECK-SD-NEXT:    add x9, sp, #72
-; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-SD-NEXT:    add x11, sp, #72
+; CHECK-SD-NEXT:    sshll v16.4s, v16.4h, #0
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]
 ; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]
-; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x11]
 ; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #0
 ; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #0
 ; CHECK-SD-NEXT:    saddw v17.4s, v17.4s, v5.4h
+; CHECK-SD-NEXT:    fmov s16, s16
 ; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    mov v18.s[0], v7.s[0]
 ; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #0
 ; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-SD-NEXT:    saddl2 v7.4s, v16.8h, v6.8h
+; CHECK-SD-NEXT:    saddl2 v18.4s, v7.8h, v6.8h
 ; CHECK-SD-NEXT:    saddl2 v5.4s, v5.8h, v4.8h
-; CHECK-SD-NEXT:    saddl v6.4s, v16.4h, v6.4h
+; CHECK-SD-NEXT:    saddl v6.4s, v7.4h, v6.4h
 ; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v4.4h
 ; CHECK-SD-NEXT:    saddl2 v17.4s, v1.8h, v0.8h
-; CHECK-SD-NEXT:    saddl2 v16.4s, v3.8h, v2.8h
-; CHECK-SD-NEXT:    saddw v1.4s, v18.4s, v1.4h
-; CHECK-SD-NEXT:    add v5.4s, v5.4s, v7.4s
-; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT:    saddl2 v7.4s, v3.8h, v2.8h
+; CHECK-SD-NEXT:    saddw v1.4s, v16.4s, v1.4h
 ; CHECK-SD-NEXT:    saddl v2.4s, v3.4h, v2.4h
-; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s
+; CHECK-SD-NEXT:    add v5.4s, v5.4s, v18.4s
+; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT:    add v6.4s, v17.4s, v7.4s
 ; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h
 ; CHECK-SD-NEXT:    add v1.4s, v4.4s, v5.4s
 ; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s
diff --git a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
index edef4a56bcd4e5..3744407c11f1c6 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-insert-zero.ll
@@ -38,12 +38,17 @@ entry:
 }
 
 define <4 x float> @testv4f32_0(<4 x float> noundef %vec) {
-; CHECK-LABEL: testv4f32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+; SD-LABEL: testv4f32_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    fmov s0, s0
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv4f32_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <4 x float> %vec, <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x float> %vecinit3
@@ -52,9 +57,7 @@ entry:
 define <4 x float> @testv4f32_1(<4 x float> noundef %vec) {
 ; SD-LABEL: testv4f32_1:
 ; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.s[0], v0.s[0]
-; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    fmov s0, s0
 ; SD-NEXT:    ret
 ;
 ; GI-LABEL: testv4f32_1:
@@ -109,24 +112,41 @@ entry:
 }
 
 define <8 x half> @testv8f16_0(<8 x half> noundef %vec) {
-; CHECK-LABEL: testv8f16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+; SD-FP16-LABEL: testv8f16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8f16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
+;
+; GI-LABEL: testv8f16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <8 x half> %vec, <8 x half> <half poison, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00, half 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x half> %vecinit3
 }
 
 define <8 x half> @testv8f16_1(<8 x half> noundef %vec) {
-; SD-LABEL: testv8f16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8f16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8f16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8f16_1:
 ; GI:       // %bb.0: // %entry
@@ -142,13 +162,18 @@ entry:
 }
 
 define <4 x half> @testv4f16_0(<4 x half> noundef %vec) {
-; SD-LABEL: testv4f16_0:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi d1, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4f16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4f16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi d1, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4f16_0:
 ; GI:       // %bb.0: // %entry
@@ -163,13 +188,18 @@ entry:
 }
 
 define <4 x half> @testv4f16_1(<4 x half> noundef %vec) {
-; SD-LABEL: testv4f16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi d1, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4f16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4f16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi d1, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4f16_1:
 ; GI:       // %bb.0: // %entry
@@ -185,24 +215,41 @@ entry:
 }
 
 define <8 x bfloat> @testv8bf16_0(<8 x bfloat> noundef %vec) {
-; CHECK-LABEL: testv8bf16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+; SD-FP16-LABEL: testv8bf16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8bf16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
+;
+; GI-LABEL: testv8bf16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <8 x bfloat> %vec, <8 x bfloat> <bfloat poison, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00, bfloat 0.000000e+00>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x bfloat> %vecinit3
 }
 
 define <8 x bfloat> @testv8bf16_1(<8 x bfloat> noundef %vec) {
-; SD-LABEL: testv8bf16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8bf16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8bf16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8bf16_1:
 ; GI:       // %bb.0: // %entry
@@ -218,13 +265,18 @@ entry:
 }
 
 define <4 x bfloat> @testv4bf16_0(<4 x bfloat> noundef %vec) {
-; SD-LABEL: testv4bf16_0:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi d1, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4bf16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4bf16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi d1, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4bf16_0:
 ; GI:       // %bb.0: // %entry
@@ -239,13 +291,18 @@ entry:
 }
 
 define <4 x bfloat> @testv4bf16_1(<4 x bfloat> noundef %vec) {
-; SD-LABEL: testv4bf16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi d1, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4bf16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4bf16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi d1, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4bf16_1:
 ; GI:       // %bb.0: // %entry
@@ -294,12 +351,17 @@ entry:
 }
 
 define <4 x i32> @testv4i32_0(<4 x i32> noundef %vec) {
-; CHECK-LABEL: testv4i32_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+; SD-LABEL: testv4i32_0:
+; SD:       // %bb.0: // %entry
+; SD-NEXT:    fmov s0, s0
+; SD-NEXT:    ret
+;
+; GI-LABEL: testv4i32_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    mov v1.s[0], v0.s[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <4 x i32> %vec, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i32> %vecinit3
@@ -308,9 +370,7 @@ entry:
 define <4 x i32> @testv4i32_1(<4 x i32> noundef %vec) {
 ; SD-LABEL: testv4i32_1:
 ; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.s[0], v0.s[0]
-; SD-NEXT:    mov v0.16b, v1.16b
+; SD-NEXT:    fmov s0, s0
 ; SD-NEXT:    ret
 ;
 ; GI-LABEL: testv4i32_1:
@@ -364,24 +424,41 @@ entry:
 }
 
 define <8 x i16> @testv8i16_0(<8 x i16> noundef %vec) {
-; CHECK-LABEL: testv8i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ret
+; SD-FP16-LABEL: testv8i16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8i16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
+;
+; GI-LABEL: testv8i16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    mov v0.16b, v1.16b
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <8 x i16> %vec, <8 x i16> <i16 poison, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i16> %vecinit3
 }
 
 define <8 x i16> @testv8i16_1(<8 x i16> noundef %vec) {
-; SD-LABEL: testv8i16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    mov v0.16b, v1.16b
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv8i16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv8i16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv8i16_1:
 ; GI:       // %bb.0: // %entry
@@ -395,13 +472,26 @@ entry:
 }
 
 define <4 x i16> @testv4i16_0(<4 x i16> noundef %vec) {
-; CHECK-LABEL: testv4i16_0:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov v1.h[0], v0.h[0]
-; CHECK-NEXT:    fmov d0, d1
-; CHECK-NEXT:    ret
+; SD-FP16-LABEL: testv4i16_0:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4i16_0:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
+;
+; GI-LABEL: testv4i16_0:
+; GI:       // %bb.0: // %entry
+; GI-NEXT:    movi v1.2d, #0000000000000000
+; GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; GI-NEXT:    mov v1.h[0], v0.h[0]
+; GI-NEXT:    fmov d0, d1
+; GI-NEXT:    ret
 entry:
   %vecinit3 = shufflevector <4 x i16> %vec, <4 x i16> <i16 poison, i16 0, i16 0, i16 0>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
   ret <4 x i16> %vecinit3
@@ -409,13 +499,18 @@ entry:
 
 define <4 x i16> @testv4i16_1(<4 x i16> noundef %vec) {
 ;
-; SD-LABEL: testv4i16_1:
-; SD:       // %bb.0: // %entry
-; SD-NEXT:    movi v1.2d, #0000000000000000
-; SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; SD-NEXT:    mov v1.h[0], v0.h[0]
-; SD-NEXT:    fmov d0, d1
-; SD-NEXT:    ret
+; SD-FP16-LABEL: testv4i16_1:
+; SD-FP16:       // %bb.0: // %entry
+; SD-FP16-NEXT:    fmov h0, h0
+; SD-FP16-NEXT:    ret
+;
+; SD-NOFP16-LABEL: testv4i16_1:
+; SD-NOFP16:       // %bb.0: // %entry
+; SD-NOFP16-NEXT:    movi v1.2d, #0000000000000000
+; SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; SD-NOFP16-NEXT:    mov v1.h[0], v0.h[0]
+; SD-NOFP16-NEXT:    fmov d0, d1
+; SD-NOFP16-NEXT:    ret
 ;
 ; GI-LABEL: testv4i16_1:
 ; GI:       // %bb.0: // %entry
@@ -493,5 +588,3 @@ entry:
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GI-FP16: {{.*}}
 ; GI-NOFP16: {{.*}}
-; SD-FP16: {{.*}}
-; SD-NOFP16: {{.*}}



More information about the llvm-branch-commits mailing list