[llvm] 645c0d5 - [AArch64][GlobalISel] Ensure we have a insert-subreg v4i32 GPR pattern (#142724)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 6 09:44:38 PDT 2025
Author: David Green
Date: 2025-06-06T17:44:33+01:00
New Revision: 645c0d509c43ef95b62503552c51e57c6e49f0e0
URL: https://github.com/llvm/llvm-project/commit/645c0d509c43ef95b62503552c51e57c6e49f0e0
DIFF: https://github.com/llvm/llvm-project/commit/645c0d509c43ef95b62503552c51e57c6e49f0e0.diff
LOG: [AArch64][GlobalISel] Ensure we have a insert-subreg v4i32 GPR pattern (#142724)
This is the GISel equivalent of scalar_to_vector, making sure that when
we insert into undef we use a fmov that avoids the artificial dependency
on the previous register. This adds v2i32 and v2i64 patterns too for
similar reasons.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
llvm/test/CodeGen/AArch64/aarch64-smull.ll
llvm/test/CodeGen/AArch64/abs.ll
llvm/test/CodeGen/AArch64/arm64-dup.ll
llvm/test/CodeGen/AArch64/arm64-fp128.ll
llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
llvm/test/CodeGen/AArch64/bitcast-extend.ll
llvm/test/CodeGen/AArch64/bitcast.ll
llvm/test/CodeGen/AArch64/bswap.ll
llvm/test/CodeGen/AArch64/concat-vector.ll
llvm/test/CodeGen/AArch64/ctlz.ll
llvm/test/CodeGen/AArch64/cttz.ll
llvm/test/CodeGen/AArch64/fcmp.ll
llvm/test/CodeGen/AArch64/fcopysign.ll
llvm/test/CodeGen/AArch64/fptoi.ll
llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
llvm/test/CodeGen/AArch64/freeze.ll
llvm/test/CodeGen/AArch64/fsh.ll
llvm/test/CodeGen/AArch64/icmp.ll
llvm/test/CodeGen/AArch64/insertextract.ll
llvm/test/CodeGen/AArch64/itofp.ll
llvm/test/CodeGen/AArch64/mul.ll
llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
llvm/test/CodeGen/AArch64/neon-dotreduce.ll
llvm/test/CodeGen/AArch64/phi.ll
llvm/test/CodeGen/AArch64/popcount.ll
llvm/test/CodeGen/AArch64/ptradd.ll
llvm/test/CodeGen/AArch64/rem.ll
llvm/test/CodeGen/AArch64/select_cc.ll
llvm/test/CodeGen/AArch64/sext.ll
llvm/test/CodeGen/AArch64/shift.ll
llvm/test/CodeGen/AArch64/shufflevector.ll
llvm/test/CodeGen/AArch64/trunc.ll
llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
llvm/test/CodeGen/AArch64/vecreduce-add.ll
llvm/test/CodeGen/AArch64/xtn.ll
llvm/test/CodeGen/AArch64/zext.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 727831896737d..5214b9bb3bbac 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -7269,6 +7269,13 @@ def : Pat<(v4i16 (vec_ins_or_scal_vec GPR32:$Rn)),
(SUBREG_TO_REG (i32 0),
(f32 (COPY_TO_REGCLASS GPR32:$Rn, FPR32)), ssub)>;
+def : Pat<(v2i32 (vec_ins_or_scal_vec GPR32:$Rn)),
+ (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)), GPR32:$Rn, ssub)>;
+def : Pat<(v4i32 (vec_ins_or_scal_vec GPR32:$Rn)),
+ (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), GPR32:$Rn, ssub)>;
+def : Pat<(v2i64 (vec_ins_or_scal_vec GPR64:$Rn)),
+ (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), GPR64:$Rn, dsub)>;
+
def : Pat<(v4f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
(INSERT_SUBREG (v4f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
def : Pat<(v8f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
@@ -7279,16 +7286,6 @@ def : Pat<(v4bf16 (vec_ins_or_scal_vec (bf16 FPR16:$Rn))),
def : Pat<(v8bf16 (vec_ins_or_scal_vec (bf16 FPR16:$Rn))),
(INSERT_SUBREG (v8bf16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
-def : Pat<(v2i32 (vec_ins_or_scal_vec (i32 FPR32:$Rn))),
- (v2i32 (INSERT_SUBREG (v2i32 (IMPLICIT_DEF)),
- (i32 FPR32:$Rn), ssub))>;
-def : Pat<(v4i32 (vec_ins_or_scal_vec (i32 FPR32:$Rn))),
- (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
- (i32 FPR32:$Rn), ssub))>;
-def : Pat<(v2i64 (vec_ins_or_scal_vec (i64 FPR64:$Rn))),
- (v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
- (i64 FPR64:$Rn), dsub))>;
-
def : Pat<(v4f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
(INSERT_SUBREG (v4f16 (IMPLICIT_DEF)), FPR16:$Rn, hsub)>;
def : Pat<(v8f16 (vec_ins_or_scal_vec (f16 FPR16:$Rn))),
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
index be80886ed3efe..ba53cb57c2ef2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
@@ -11,7 +11,7 @@ define i32 @bar() {
; CHECK-NEXT: movi.2d v0, #0000000000000000
; CHECK-NEXT: umov.b w8, v0[0]
; CHECK-NEXT: umov.b w9, v0[1]
-; CHECK-NEXT: mov.s v1[0], w8
+; CHECK-NEXT: fmov s1, w8
; CHECK-NEXT: umov.b w8, v0[2]
; CHECK-NEXT: mov.s v1[1], w9
; CHECK-NEXT: umov.b w9, v0[3]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
index 0115531dfb09a..22d1ccc056eb4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-build-vector-to-dup.mir
@@ -57,15 +57,12 @@ body: |
; SELECT-NEXT: %r:gpr32 = COPY $w0
; SELECT-NEXT: %q:gpr32 = COPY $w1
; SELECT-NEXT: [[DEF:%[0-9]+]]:fpr64 = IMPLICIT_DEF
+ ; SELECT-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr64 = INSERT_SUBREG [[DEF]], %r, %subreg.ssub
; SELECT-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
- ; SELECT-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[DEF]], %subreg.dsub
- ; SELECT-NEXT: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG]], 0, %r
+ ; SELECT-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[INSERT_SUBREG]], %subreg.dsub
+ ; SELECT-NEXT: [[INSvi32gpr:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, %q
; SELECT-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr]].dsub
- ; SELECT-NEXT: [[DEF2:%[0-9]+]]:fpr128 = IMPLICIT_DEF
- ; SELECT-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF2]], [[COPY]], %subreg.dsub
- ; SELECT-NEXT: [[INSvi32gpr1:%[0-9]+]]:fpr128 = INSvi32gpr [[INSERT_SUBREG1]], 1, %q
- ; SELECT-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY [[INSvi32gpr1]].dsub
- ; SELECT-NEXT: $d0 = COPY [[COPY1]]
+ ; SELECT-NEXT: $d0 = COPY [[COPY]]
; SELECT-NEXT: RET_ReallyLR implicit $d0
%r:_(s32) = COPY $w0
%q:_(s32) = COPY $w1
diff --git a/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll b/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
index 9734ab35bd6b2..7f922c0047553 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll
@@ -76,8 +76,7 @@ define <1 x i32> @test_bitf_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
; CHECK-GI-NEXT: bic w9, w9, w8
; CHECK-GI-NEXT: and w8, w8, w10
; CHECK-GI-NEXT: orr w8, w9, w8
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%neg = xor <1 x i32> %C, <i32 -1>
%and = and <1 x i32> %neg, %B
diff --git a/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll b/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
index 45ad4b07ff66f..b8eb8269d605c 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
@@ -76,8 +76,7 @@ define <1 x i32> @test_bit_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
; CHECK-GI-NEXT: and w9, w8, w9
; CHECK-GI-NEXT: bic w8, w10, w8
; CHECK-GI-NEXT: orr w8, w9, w8
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%and = and <1 x i32> %C, %B
%neg = xor <1 x i32> %C, <i32 -1>
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
index 418113a4e4e09..8655bb1292ef7 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-matrix-umull-smull.ll
@@ -204,7 +204,7 @@ define void @matrix_mul_double_shuffle(i32 %N, ptr nocapture %C, ptr nocapture r
; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-GI-NEXT: ldrh w9, [x2], #16
; CHECK-GI-NEXT: subs x8, x8, #8
-; CHECK-GI-NEXT: mov v2.s[0], w9
+; CHECK-GI-NEXT: fmov s2, w9
; CHECK-GI-NEXT: mov w9, w0
; CHECK-GI-NEXT: add w0, w0, #8
; CHECK-GI-NEXT: lsl x9, x9, #2
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index d6fd4c4110a12..2f23a32c36a9f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -2282,14 +2282,14 @@ define <2 x i64> @asr(<2 x i64> %a, <2 x i64> %b) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #32
; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #32
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
%x = ashr <2 x i64> %a, <i64 32, i64 32>
%y = ashr <2 x i64> %b, <i64 32, i64 32>
@@ -2317,14 +2317,14 @@ define <2 x i64> @asr_const(<2 x i64> %a, <2 x i64> %b) {
; CHECK-GI-NEXT: adrp x8, .LCPI81_0
; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #32
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI81_0]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
%x = ashr <2 x i64> %a, <i64 32, i64 32>
%z = mul nsw <2 x i64> %x, <i64 31, i64 31>
@@ -2799,14 +2799,14 @@ define <2 x i64> @sdistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-GI-NEXT: saddl v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d2
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v2.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d2
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v2.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2838,14 +2838,14 @@ define <2 x i64> @sdistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {
; CHECK-GI-NEXT: sshll v1.2d, v1.2s, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI101_0]
; CHECK-GI-NEXT: saddw v0.2d, v2.2d, v0.2s
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x11, v1.d[1]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov x10, v0.d[1]
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2875,14 +2875,14 @@ define <2 x i64> @sdistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-GI-NEXT: adrp x8, .LCPI102_0
; CHECK-GI-NEXT: saddl v0.2d, v0.2s, v1.2s
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI102_0]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = sext <2 x i32> %src1 to <2 x i64>
@@ -2909,14 +2909,14 @@ define <2 x i64> @udistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ushll v2.2d, v2.2s, #0
; CHECK-GI-NEXT: uaddl v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d2
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v2.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d2
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v2.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = zext <2 x i32> %src1 to <2 x i64>
@@ -2948,14 +2948,14 @@ define <2 x i64> @udistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {
; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI104_0]
; CHECK-GI-NEXT: uaddw v0.2d, v2.2d, v0.2s
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x11, v1.d[1]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov x10, v0.d[1]
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = zext <2 x i32> %src1 to <2 x i64>
@@ -2985,14 +2985,14 @@ define <2 x i64> @udistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-GI-NEXT: adrp x8, .LCPI105_0
; CHECK-GI-NEXT: uaddl v0.2d, v0.2s, v1.2s
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI105_0]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%4 = zext <2 x i32> %src1 to <2 x i64>
diff --git a/llvm/test/CodeGen/AArch64/abs.ll b/llvm/test/CodeGen/AArch64/abs.ll
index d501d9ed24547..0f56d25a47b2a 100644
--- a/llvm/test/CodeGen/AArch64/abs.ll
+++ b/llvm/test/CodeGen/AArch64/abs.ll
@@ -247,8 +247,7 @@ define <1 x i32> @abs_v1i32(<1 x i32> %a){
; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: cmp w8, #0
; CHECK-GI-NEXT: cneg w8, w9, le
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
entry:
%res = call <1 x i32> @llvm.abs.v1i32(<1 x i32> %a, i1 0)
diff --git a/llvm/test/CodeGen/AArch64/arm64-dup.ll b/llvm/test/CodeGen/AArch64/arm64-dup.ll
index 4c28ea7592202..c279cf0f241d2 100644
--- a/llvm/test/CodeGen/AArch64/arm64-dup.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-dup.ll
@@ -334,40 +334,25 @@ entry:
}
define <2 x i32> @f(i32 %a, i32 %b) nounwind readnone {
-; CHECK-SD-LABEL: f:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov.s v0[1], w1
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: f:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov.s v0[0], w0
-; CHECK-GI-NEXT: mov.s v0[1], w1
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: f:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: mov.s v0[1], w1
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
%vecinit = insertelement <2 x i32> undef, i32 %a, i32 0
%vecinit1 = insertelement <2 x i32> %vecinit, i32 %b, i32 1
ret <2 x i32> %vecinit1
}
define <4 x i32> @g(i32 %a, i32 %b) nounwind readnone {
-; CHECK-SD-LABEL: g:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov.s v0[1], w1
-; CHECK-SD-NEXT: mov.s v0[2], w1
-; CHECK-SD-NEXT: mov.s v0[3], w0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: g:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov.s v0[0], w0
-; CHECK-GI-NEXT: mov.s v0[1], w1
-; CHECK-GI-NEXT: mov.s v0[2], w1
-; CHECK-GI-NEXT: mov.s v0[3], w0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: g:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: mov.s v0[1], w1
+; CHECK-NEXT: mov.s v0[2], w1
+; CHECK-NEXT: mov.s v0[3], w0
+; CHECK-NEXT: ret
%vecinit = insertelement <4 x i32> undef, i32 %a, i32 0
%vecinit1 = insertelement <4 x i32> %vecinit, i32 %b, i32 1
%vecinit2 = insertelement <4 x i32> %vecinit1, i32 %b, i32 2
@@ -376,17 +361,11 @@ define <4 x i32> @g(i32 %a, i32 %b) nounwind readnone {
}
define <2 x i64> @h(i64 %a, i64 %b) nounwind readnone {
-; CHECK-SD-LABEL: h:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov d0, x0
-; CHECK-SD-NEXT: mov.d v0[1], x1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: h:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov.d v0[0], x0
-; CHECK-GI-NEXT: mov.d v0[1], x1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov d0, x0
+; CHECK-NEXT: mov.d v0[1], x1
+; CHECK-NEXT: ret
%vecinit = insertelement <2 x i64> undef, i64 %a, i32 0
%vecinit1 = insertelement <2 x i64> %vecinit, i64 %b, i32 1
ret <2 x i64> %vecinit1
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp128.ll b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
index 7eb26096ed156..a75f6419d5a5a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
@@ -618,7 +618,7 @@ define <2 x i32> @vec_fptosi_32(<2 x fp128> %val) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixtfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -661,7 +661,7 @@ define <2 x i64> @vec_fptosi_64(<2 x fp128> %val) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov x19, x0
; CHECK-GI-NEXT: bl __fixtfdi
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.d[1], x0
; CHECK-GI-NEXT: add sp, sp, #32
@@ -702,7 +702,7 @@ define <2 x i32> @vec_fptoui_32(<2 x fp128> %val) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -745,7 +745,7 @@ define <2 x i64> @vec_fptoui_64(<2 x fp128> %val) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov x19, x0
; CHECK-GI-NEXT: bl __fixunstfdi
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.d[1], x0
; CHECK-GI-NEXT: add sp, sp, #32
@@ -977,7 +977,7 @@ define <2 x i1> @vec_setcc1(<2 x fp128> %lhs, <2 x fp128> %rhs) {
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w19, le
; CHECK-GI-NEXT: bl __letf2
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w8, le
; CHECK-GI-NEXT: ldp x30, x19, [sp, #32] // 16-byte Folded Reload
@@ -1032,7 +1032,7 @@ define <2 x i1> @vec_setcc2(<2 x fp128> %lhs, <2 x fp128> %rhs) {
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w19, gt
; CHECK-GI-NEXT: bl __letf2
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w8, gt
; CHECK-GI-NEXT: ldp x30, x19, [sp, #32] // 16-byte Folded Reload
@@ -1109,7 +1109,7 @@ define <2 x i1> @vec_setcc3(<2 x fp128> %lhs, <2 x fp128> %rhs) {
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w20, eq
; CHECK-GI-NEXT: bl __unordtf2
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-GI-NEXT: cset w8, ne
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 51f1351a5edf4..ddd8a72618b1e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -1198,44 +1198,28 @@ define <8 x i16> @scalar_to_vector.v8i16(i16 %a) {
}
define <2 x i32> @scalar_to_vector.v2i32(i32 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v2i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v2i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: scalar_to_vector.v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: ret
%b = insertelement <2 x i32> undef, i32 %a, i32 0
ret <2 x i32> %b
}
define <4 x i32> @scalar_to_vector.v4i32(i32 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v4i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v4i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: scalar_to_vector.v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: ret
%b = insertelement <4 x i32> undef, i32 %a, i32 0
ret <4 x i32> %b
}
define <2 x i64> @scalar_to_vector.v2i64(i64 %a) {
-; CHECK-SD-LABEL: scalar_to_vector.v2i64:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov d0, x0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: scalar_to_vector.v2i64:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov v0.d[0], x0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: scalar_to_vector.v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov d0, x0
+; CHECK-NEXT: ret
%b = insertelement <2 x i64> undef, i64 %a, i32 0
ret <2 x i64> %b
}
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
index ca6bb8360de59..bdbebd8726fde 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div-cte.ll
@@ -139,7 +139,7 @@ define <4 x i32> @div32xi4(<4 x i32> %x) {
; CHECK-GI-NEXT: mov w12, v0.s[3]
; CHECK-GI-NEXT: sdiv w9, w9, w8
; CHECK-GI-NEXT: sdiv w10, w10, w8
-; CHECK-GI-NEXT: mov v0.s[0], w9
+; CHECK-GI-NEXT: fmov s0, w9
; CHECK-GI-NEXT: sdiv w11, w11, w8
; CHECK-GI-NEXT: mov v0.s[1], w10
; CHECK-GI-NEXT: sdiv w8, w12, w8
@@ -240,14 +240,14 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %a) {
; CHECK-GI-LABEL: udiv_v2i64:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov x8, #9363 // =0x2493
-; CHECK-GI-NEXT: fmov x9, d0
-; CHECK-GI-NEXT: mov x10, v0.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: mov x9, v0.d[1]
; CHECK-GI-NEXT: movk x8, #37449, lsl #16
; CHECK-GI-NEXT: movk x8, #18724, lsl #32
; CHECK-GI-NEXT: movk x8, #9362, lsl #48
-; CHECK-GI-NEXT: umulh x9, x9, x8
-; CHECK-GI-NEXT: umulh x8, x10, x8
-; CHECK-GI-NEXT: mov v1.d[0], x9
+; CHECK-GI-NEXT: umulh x10, x10, x8
+; CHECK-GI-NEXT: umulh x8, x9, x8
+; CHECK-GI-NEXT: fmov d1, x10
; CHECK-GI-NEXT: mov v1.d[1], x8
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: usra v1.2d, v0.2d, #1
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll b/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
index bbea8f7b93f02..7b439dd36c425 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll
@@ -405,25 +405,15 @@ define i16 @test_sqrdmlah_v1i16(i16 %acc, i16 %x, i16 %y) {
}
define i32 @test_sqrdmlah_v1i32(i32 %acc, i32 %x, i32 %y) {
-; CHECK-SD-LABEL: test_sqrdmlah_v1i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w1
-; CHECK-SD-NEXT: fmov s1, w2
-; CHECK-SD-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: sqadd v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_sqrdmlah_v1i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov v0.s[0], w1
-; CHECK-GI-NEXT: mov v1.s[0], w2
-; CHECK-GI-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: mov v1.s[0], w0
-; CHECK-GI-NEXT: sqadd v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: fmov w0, s0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: test_sqrdmlah_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w1
+; CHECK-NEXT: fmov s1, w2
+; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: sqadd v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
%x_vec = insertelement <4 x i32> undef, i32 %x, i64 0
%y_vec = insertelement <4 x i32> undef, i32 %y, i64 0
%prod_vec = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x_vec, <4 x i32> %y_vec)
@@ -454,25 +444,15 @@ define i16 @test_sqrdmlsh_v1i16(i16 %acc, i16 %x, i16 %y) {
}
define i32 @test_sqrdmlsh_v1i32(i32 %acc, i32 %x, i32 %y) {
-; CHECK-SD-LABEL: test_sqrdmlsh_v1i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w1
-; CHECK-SD-NEXT: fmov s1, w2
-; CHECK-SD-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: sqsub v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_sqrdmlsh_v1i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: mov v0.s[0], w1
-; CHECK-GI-NEXT: mov v1.s[0], w2
-; CHECK-GI-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: mov v1.s[0], w0
-; CHECK-GI-NEXT: sqsub v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: fmov w0, s0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: test_sqrdmlsh_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov s0, w1
+; CHECK-NEXT: fmov s1, w2
+; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: sqsub v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
%x_vec = insertelement <4 x i32> undef, i32 %x, i64 0
%y_vec = insertelement <4 x i32> undef, i32 %y, i64 0
%prod_vec = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x_vec, <4 x i32> %y_vec)
diff --git a/llvm/test/CodeGen/AArch64/bitcast-extend.ll b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
index 3133d0efb4b9b..f62303ecea663 100644
--- a/llvm/test/CodeGen/AArch64/bitcast-extend.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast-extend.ll
@@ -84,20 +84,20 @@ define <4 x i64> @z_i32_v4i64(i32 %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: mov b1, v0.b[2]
-; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: mov b2, v0.b[1]
; CHECK-GI-NEXT: mov b3, v0.b[3]
-; CHECK-GI-NEXT: ubfx x8, x8, #0, #8
+; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: fmov w8, s2
-; CHECK-GI-NEXT: ubfx x9, x9, #0, #8
; CHECK-GI-NEXT: ubfx x8, x8, #0, #8
-; CHECK-GI-NEXT: mov v1.d[0], x9
-; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: fmov w10, s2
+; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: fmov d0, x8
; CHECK-GI-NEXT: ubfx x9, x9, #0, #8
-; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: ubfx x10, x10, #0, #8
+; CHECK-GI-NEXT: ubfx x11, x11, #0, #8
+; CHECK-GI-NEXT: fmov d1, x9
+; CHECK-GI-NEXT: mov v0.d[1], x10
+; CHECK-GI-NEXT: mov v1.d[1], x11
; CHECK-GI-NEXT: ret
%b = bitcast i32 %x to <4 x i8>
%e = zext <4 x i8> %b to <4 x i64>
@@ -188,20 +188,20 @@ define <4 x i64> @s_i32_v4i64(i32 %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: mov b1, v0.b[2]
-; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: mov b2, v0.b[1]
; CHECK-GI-NEXT: mov b3, v0.b[3]
-; CHECK-GI-NEXT: sxtb x8, w8
+; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: fmov w8, s2
-; CHECK-GI-NEXT: sxtb x9, w9
; CHECK-GI-NEXT: sxtb x8, w8
-; CHECK-GI-NEXT: mov v1.d[0], x9
-; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: fmov w10, s2
+; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: fmov d0, x8
; CHECK-GI-NEXT: sxtb x9, w9
-; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: sxtb x10, w10
+; CHECK-GI-NEXT: sxtb x11, w11
+; CHECK-GI-NEXT: fmov d1, x9
+; CHECK-GI-NEXT: mov v0.d[1], x10
+; CHECK-GI-NEXT: mov v1.d[1], x11
; CHECK-GI-NEXT: ret
%b = bitcast i32 %x to <4 x i8>
%e = sext <4 x i8> %b to <4 x i64>
diff --git a/llvm/test/CodeGen/AArch64/bitcast.ll b/llvm/test/CodeGen/AArch64/bitcast.ll
index d54cc4adb81b3..38cec0d71a945 100644
--- a/llvm/test/CodeGen/AArch64/bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast.ll
@@ -13,7 +13,7 @@ define <4 x i16> @foo1(<2 x i32> %a) {
; CHECK-GI-LABEL: foo1:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #58712 // =0xe558
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: zip1 v0.2s, v1.2s, v0.2s
; CHECK-GI-NEXT: rev32 v0.4h, v0.4h
; CHECK-GI-NEXT: ret
@@ -33,7 +33,7 @@ define <4 x i16> @foo2(<2 x i32> %a) {
; CHECK-GI-LABEL: foo2:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #712 // =0x2c8
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: zip1 v0.2s, v1.2s, v0.2s
; CHECK-GI-NEXT: rev32 v0.4h, v0.4h
; CHECK-GI-NEXT: ret
@@ -133,7 +133,6 @@ define <2 x i16> @bitcast_i32_v2i16(i32 %a, i32 %b){
; CHECK-GI-NEXT: add w8, w0, w1
; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: mov h1, v0.h[1]
-; CHECK-GI-NEXT: mov v0.s[0], w8
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -449,8 +448,6 @@ define <2 x i16> @bitcast_v4i8_v2i16(<4 x i8> %a, <4 x i8> %b){
; CHECK-GI-NEXT: add v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT: mov h1, v0.h[1]
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -503,12 +500,10 @@ define <4 x i64> @bitcast_v8i32_v4i64(<8 x i32> %a, <8 x i32> %b){
;
; CHECK-GI-LABEL: bitcast_v8i32_v4i64:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: add v2.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: add v3.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT: mov x8, v2.d[1]
-; CHECK-GI-NEXT: mov x9, v3.d[1]
-; CHECK-GI-NEXT: mov v0.d[0], v2.d[0]
-; CHECK-GI-NEXT: mov v1.d[0], v3.d[0]
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: ret
@@ -564,12 +559,10 @@ define <4 x i64> @bitcast_v16i16_v4i64(<16 x i16> %a, <16 x i16> %b){
;
; CHECK-GI-LABEL: bitcast_v16i16_v4i64:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: add v2.8h, v0.8h, v2.8h
-; CHECK-GI-NEXT: add v3.8h, v1.8h, v3.8h
-; CHECK-GI-NEXT: mov x8, v2.d[1]
-; CHECK-GI-NEXT: mov x9, v3.d[1]
-; CHECK-GI-NEXT: mov v0.d[0], v2.d[0]
-; CHECK-GI-NEXT: mov v1.d[0], v3.d[0]
+; CHECK-GI-NEXT: add v0.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT: add v1.8h, v1.8h, v3.8h
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: ret
@@ -606,18 +599,14 @@ define <8 x i64> @bitcast_v16i32_v8i64(<16 x i32> %a, <16 x i32> %b){
;
; CHECK-GI-LABEL: bitcast_v16i32_v8i64:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: add v4.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: add v5.4s, v1.4s, v5.4s
-; CHECK-GI-NEXT: add v6.4s, v2.4s, v6.4s
-; CHECK-GI-NEXT: add v7.4s, v3.4s, v7.4s
-; CHECK-GI-NEXT: mov x8, v4.d[1]
-; CHECK-GI-NEXT: mov x9, v5.d[1]
-; CHECK-GI-NEXT: mov x10, v6.d[1]
-; CHECK-GI-NEXT: mov x11, v7.d[1]
-; CHECK-GI-NEXT: mov v0.d[0], v4.d[0]
-; CHECK-GI-NEXT: mov v1.d[0], v5.d[0]
-; CHECK-GI-NEXT: mov v2.d[0], v6.d[0]
-; CHECK-GI-NEXT: mov v3.d[0], v7.d[0]
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v6.4s
+; CHECK-GI-NEXT: add v3.4s, v3.4s, v7.4s
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mov x10, v2.d[1]
+; CHECK-GI-NEXT: mov x11, v3.d[1]
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: mov v2.d[1], x10
diff --git a/llvm/test/CodeGen/AArch64/bswap.ll b/llvm/test/CodeGen/AArch64/bswap.ll
index e86f55d63f754..898958fb4993f 100644
--- a/llvm/test/CodeGen/AArch64/bswap.ll
+++ b/llvm/test/CodeGen/AArch64/bswap.ll
@@ -209,8 +209,7 @@ define <1 x i32> @bswap_v1i32(<1 x i32> %a){
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: rev w8, w8
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
entry:
%res = call <1 x i32> @llvm.bswap.v1i32(<1 x i32> %a)
diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll
index 05a40453833ee..acf15f1bd1178 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -32,10 +32,8 @@ define <8 x i8> @concat2(<4 x i8> %A, <4 x i8> %B) {
;
; CHECK-GI-LABEL: concat2:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT: uzp1 v1.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -63,10 +61,8 @@ define <4 x i16> @concat4(<2 x i16> %A, <2 x i16> %B) {
;
; CHECK-GI-LABEL: concat4:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: uzp1 v1.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -129,9 +125,8 @@ define <4 x half> @concat9(<2 x half> %A, <2 x half> %B) {
;
; CHECK-GI-LABEL: concat9:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
@@ -168,9 +163,8 @@ define <8 x i16> @concat_v8s16_v2s16(ptr %ptr) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: ldrh w8, [x0]
; CHECK-GI-NEXT: ldrh w9, [x0, #2]
-; CHECK-GI-NEXT: fmov s1, w8
-; CHECK-GI-NEXT: mov v1.h[1], w9
-; CHECK-GI-NEXT: mov v0.s[0], v1.s[0]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: mov v0.h[1], w9
; CHECK-GI-NEXT: ret
%a = load <2 x i16>, ptr %ptr
%b = shufflevector <2 x i16> %a, <2 x i16> %a, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -220,15 +214,13 @@ define <16 x i8> @concat_v16s8_v4s8_reg(<4 x i8> %A, <4 x i8> %B, <4 x i8> %C, <
;
; CHECK-GI-LABEL: concat_v16s8_v4s8_reg:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT: uzp1 v1.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: uzp1 v2.8b, v2.8b, v0.8b
; CHECK-GI-NEXT: mov v0.s[1], w8
-; CHECK-GI-NEXT: uzp1 v1.8b, v3.8b, v0.8b
; CHECK-GI-NEXT: fmov w8, s2
+; CHECK-GI-NEXT: uzp1 v1.8b, v3.8b, v0.8b
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[3], w8
@@ -253,15 +245,13 @@ define <8 x i16> @concat_v8s16_v2s16_reg(<2 x i16> %A, <2 x i16> %B, <2 x i16> %
;
; CHECK-GI-LABEL: concat_v8s16_v2s16_reg:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: uzp1 v1.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: uzp1 v2.4h, v2.4h, v0.4h
; CHECK-GI-NEXT: mov v0.s[1], w8
-; CHECK-GI-NEXT: uzp1 v1.4h, v3.4h, v0.4h
; CHECK-GI-NEXT: fmov w8, s2
+; CHECK-GI-NEXT: uzp1 v1.4h, v3.4h, v0.4h
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: mov v0.s[3], w8
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
index a4863d1f74200..b1c6e24c30a7d 100644
--- a/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -294,11 +294,11 @@ define <2 x i64> @v2i64(<2 x i64> %d) {
;
; CHECK-GI-LABEL: v2i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov x9, v0.d[1]
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: clz x9, x9
+; CHECK-GI-NEXT: fmov d0, x9
; CHECK-GI-NEXT: clz x8, x8
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: clz x8, x9
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
@@ -411,16 +411,16 @@ define <4 x i64> @v4i64(<4 x i64> %d) {
;
; CHECK-GI-LABEL: v4i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov x9, d0
-; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
; CHECK-GI-NEXT: mov x8, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
-; CHECK-GI-NEXT: clz x9, x9
+; CHECK-GI-NEXT: mov x9, v1.d[1]
; CHECK-GI-NEXT: clz x10, x10
-; CHECK-GI-NEXT: mov v0.d[0], x9
-; CHECK-GI-NEXT: mov v1.d[0], x10
+; CHECK-GI-NEXT: clz x11, x11
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: fmov d1, x11
; CHECK-GI-NEXT: clz x8, x8
-; CHECK-GI-NEXT: clz x9, x11
+; CHECK-GI-NEXT: clz x9, x9
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index 60125f8a19811..93ac97e20dabd 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -235,15 +235,15 @@ define void @v2i16(ptr %p1) {
; CHECK-GI-LABEL: v2i16:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #65535 // =0xffff
-; CHECK-GI-NEXT: ld1 { v1.h }[0], [x0]
-; CHECK-GI-NEXT: ldr h2, [x0, #2]
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: ld1 { v0.h }[0], [x0]
+; CHECK-GI-NEXT: ldr h1, [x0, #2]
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v2.s[1], w8
; CHECK-GI-NEXT: add x8, x0, #2
-; CHECK-GI-NEXT: eor v2.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT: add v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT: and v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT: eor v1.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT: add v0.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT: and v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: cnt v0.8b, v0.8b
; CHECK-GI-NEXT: uaddlp v0.4h, v0.8b
@@ -418,7 +418,7 @@ define <3 x i32> @v3i32(<3 x i32> %d) {
; CHECK-GI-LABEL: v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: mov v1.s[1], w8
; CHECK-GI-NEXT: mov v1.s[2], w8
; CHECK-GI-NEXT: eor v2.16b, v0.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll
index b9d65fa0314c4..f33f57e8751ca 100644
--- a/llvm/test/CodeGen/AArch64/fcmp.ll
+++ b/llvm/test/CodeGen/AArch64/fcmp.ll
@@ -654,7 +654,7 @@ define <2 x double> @v2f128_double(<2 x fp128> %a, <2 x fp128> %b, <2 x double>
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w19, lt
; CHECK-GI-NEXT: bl __lttf2
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w8, lt
; CHECK-GI-NEXT: ldp q2, q1, [sp, #32] // 32-byte Folded Reload
@@ -761,29 +761,29 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: cset w22, lt
; CHECK-GI-NEXT: bl __lttf2
+; CHECK-GI-NEXT: ldp q0, q2, [sp, #64] // 32-byte Folded Reload
; CHECK-GI-NEXT: sbfx x8, x21, #0, #1
-; CHECK-GI-NEXT: ldp q3, q2, [sp, #64] // 32-byte Folded Reload
+; CHECK-GI-NEXT: ldp q4, q3, [sp, #96] // 32-byte Folded Reload
+; CHECK-GI-NEXT: sbfx x9, x22, #0, #1
+; CHECK-GI-NEXT: fmov d1, x8
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: ldr x30, [sp, #128] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v1.d[0], x8
-; CHECK-GI-NEXT: sbfx x8, x22, #0, #1
-; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-GI-NEXT: ldp q4, q3, [sp, #96] // 32-byte Folded Reload
-; CHECK-GI-NEXT: ldp x22, x21, [sp, #144] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.d[1], x8
-; CHECK-GI-NEXT: mov v1.d[1], x8
-; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT: mov v2.d[1], v0.d[0]
+; CHECK-GI-NEXT: fmov d0, x8
; CHECK-GI-NEXT: cset w8, lt
+; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT: sbfx x8, x8, #0, #1
-; CHECK-GI-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-GI-NEXT: bic v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v0.d[1], x9
; CHECK-GI-NEXT: and x9, x19, x8
; CHECK-GI-NEXT: bic x8, x20, x8
; CHECK-GI-NEXT: ldp x20, x19, [sp, #160] // 16-byte Folded Reload
; CHECK-GI-NEXT: orr x8, x9, x8
-; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT: bic v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT: fmov d2, x8
+; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: add sp, sp, #176
@@ -929,22 +929,22 @@ define <3 x i32> @v3f64_i32(<3 x double> %a, <3 x double> %b, <3 x i32> %d, <3 x
; CHECK-GI-NEXT: fcmp d2, d5
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: cset w9, mi
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
-; CHECK-GI-NEXT: fcmgt v0.2d, v3.2d, v0.2d
; CHECK-GI-NEXT: mov v1.s[1], w8
-; CHECK-GI-NEXT: mov v3.s[0], w9
-; CHECK-GI-NEXT: xtn v0.2s, v0.2d
+; CHECK-GI-NEXT: fmov s2, w9
+; CHECK-GI-NEXT: fcmgt v0.2d, v3.2d, v0.2d
; CHECK-GI-NEXT: mov v1.s[2], w8
-; CHECK-GI-NEXT: mov v3.s[1], w9
+; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT: xtn v0.2s, v0.2d
; CHECK-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-GI-NEXT: mov v3.s[2], w9
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: mov v2.s[1], w8
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT: neg v1.4s, v1.4s
+; CHECK-GI-NEXT: mov v2.s[2], w8
; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: eor v1.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT: eor v1.16b, v0.16b, v2.16b
; CHECK-GI-NEXT: and v0.16b, v6.16b, v0.16b
; CHECK-GI-NEXT: and v1.16b, v7.16b, v1.16b
; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
@@ -999,18 +999,18 @@ define <3 x float> @v3f32_float(<3 x float> %a, <3 x float> %b, <3 x float> %d,
; CHECK-GI-LABEL: v3f32_float:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #31 // =0x1f
-; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
; CHECK-GI-NEXT: fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov v5.s[0], w9
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: mov v5.s[1], w9
; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: mov v1.s[1], w8
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: neg v1.4s, v4.4s
-; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: mov v1.s[2], w8
+; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
@@ -1077,18 +1077,18 @@ define <3 x i32> @v3f32_i32(<3 x float> %a, <3 x float> %b, <3 x i32> %d, <3 x i
; CHECK-GI-LABEL: v3f32_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #31 // =0x1f
-; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
; CHECK-GI-NEXT: fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov v5.s[0], w9
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: mov v5.s[1], w9
; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: mov v1.s[1], w8
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: neg v1.4s, v4.4s
-; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: mov v1.s[2], w8
+; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
@@ -1449,49 +1449,49 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32
; CHECK-GI-NOFP16-NEXT: mov v2.h[0], v0.h[4]
; CHECK-GI-NOFP16-NEXT: mov v3.h[0], v1.h[4]
; CHECK-GI-NOFP16-NEXT: mov w8, #31 // =0x1f
-; CHECK-GI-NOFP16-NEXT: mov v4.s[0], w8
-; CHECK-GI-NOFP16-NEXT: mov w9, #-1 // =0xffffffff
-; CHECK-GI-NOFP16-NEXT: mov v5.s[0], w0
-; CHECK-GI-NOFP16-NEXT: mov v6.s[0], w9
-; CHECK-GI-NOFP16-NEXT: mov v7.s[0], w7
-; CHECK-GI-NOFP16-NEXT: ldr s16, [sp]
-; CHECK-GI-NOFP16-NEXT: ldr s17, [sp, #24]
-; CHECK-GI-NOFP16-NEXT: ldr s18, [sp, #32]
+; CHECK-GI-NOFP16-NEXT: fmov s4, w8
+; CHECK-GI-NOFP16-NEXT: fmov s7, w0
+; CHECK-GI-NOFP16-NEXT: ldr s5, [sp]
+; CHECK-GI-NOFP16-NEXT: fmov s16, w7
+; CHECK-GI-NOFP16-NEXT: fmov s18, w4
+; CHECK-GI-NOFP16-NEXT: ldr s17, [sp, #32]
+; CHECK-GI-NOFP16-NEXT: ldr s6, [sp, #8]
; CHECK-GI-NOFP16-NEXT: mov v2.h[1], v0.h[5]
; CHECK-GI-NOFP16-NEXT: mov v3.h[1], v1.h[5]
; CHECK-GI-NOFP16-NEXT: mov v4.s[1], w8
-; CHECK-GI-NOFP16-NEXT: mov v5.s[1], w1
-; CHECK-GI-NOFP16-NEXT: mov v17.s[1], v18.s[0]
-; CHECK-GI-NOFP16-NEXT: mov v6.s[1], w9
-; CHECK-GI-NOFP16-NEXT: mov v7.s[1], v16.s[0]
-; CHECK-GI-NOFP16-NEXT: ldr s16, [sp, #8]
+; CHECK-GI-NOFP16-NEXT: mov v7.s[1], w1
+; CHECK-GI-NOFP16-NEXT: mov v16.s[1], v5.s[0]
+; CHECK-GI-NOFP16-NEXT: ldr s5, [sp, #24]
+; CHECK-GI-NOFP16-NEXT: mov v18.s[1], w5
+; CHECK-GI-NOFP16-NEXT: mov v5.s[1], v17.s[0]
; CHECK-GI-NOFP16-NEXT: mov v2.h[2], v0.h[6]
; CHECK-GI-NOFP16-NEXT: mov v3.h[2], v1.h[6]
-; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h
; CHECK-GI-NOFP16-NEXT: mov v4.s[2], w8
+; CHECK-GI-NOFP16-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT: mov v5.s[2], w2
-; CHECK-GI-NOFP16-NEXT: mov v6.s[2], w9
-; CHECK-GI-NOFP16-NEXT: mov v7.s[2], v16.s[0]
-; CHECK-GI-NOFP16-NEXT: ldr s16, [sp, #40]
+; CHECK-GI-NOFP16-NEXT: mov v7.s[2], w2
+; CHECK-GI-NOFP16-NEXT: mov v16.s[2], v6.s[0]
+; CHECK-GI-NOFP16-NEXT: ldr s6, [sp, #40]
+; CHECK-GI-NOFP16-NEXT: mov v18.s[2], w6
; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT: mov v17.s[2], v16.s[0]
+; CHECK-GI-NOFP16-NEXT: mov v5.s[2], v6.s[0]
; CHECK-GI-NOFP16-NEXT: fcmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT: mov v5.s[3], w3
+; CHECK-GI-NOFP16-NEXT: mov v7.s[3], w3
; CHECK-GI-NOFP16-NEXT: fcmgt v2.4s, v3.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT: mov v3.s[0], w4
+; CHECK-GI-NOFP16-NEXT: fmov s3, w8
+; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w8
; CHECK-GI-NOFP16-NEXT: ushl v2.4s, v2.4s, v4.4s
; CHECK-GI-NOFP16-NEXT: neg v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w5
+; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w8
; CHECK-GI-NOFP16-NEXT: sshl v2.4s, v2.4s, v4.4s
; CHECK-GI-NOFP16-NEXT: ldr s4, [sp, #16]
-; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w6
-; CHECK-GI-NOFP16-NEXT: mov v7.s[3], v4.s[0]
-; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v6.16b
-; CHECK-GI-NOFP16-NEXT: and v2.16b, v3.16b, v2.16b
-; CHECK-GI-NOFP16-NEXT: and v1.16b, v17.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT: bsl v0.16b, v5.16b, v7.16b
+; CHECK-GI-NOFP16-NEXT: mov v16.s[3], v4.s[0]
+; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v3.16b
+; CHECK-GI-NOFP16-NEXT: and v2.16b, v18.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT: bsl v0.16b, v7.16b, v16.16b
+; CHECK-GI-NOFP16-NEXT: and v1.16b, v5.16b, v1.16b
; CHECK-GI-NOFP16-NEXT: orr v1.16b, v2.16b, v1.16b
; CHECK-GI-NOFP16-NEXT: mov s2, v0.s[1]
; CHECK-GI-NOFP16-NEXT: mov s3, v0.s[2]
@@ -1510,60 +1510,60 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32
; CHECK-GI-FP16-LABEL: v7f16_i32:
; CHECK-GI-FP16: // %bb.0: // %entry
; CHECK-GI-FP16-NEXT: fcmgt v0.8h, v1.8h, v0.8h
-; CHECK-GI-FP16-NEXT: mov w9, #31 // =0x1f
-; CHECK-GI-FP16-NEXT: mov v4.s[0], w0
-; CHECK-GI-FP16-NEXT: mov v2.s[0], w9
-; CHECK-GI-FP16-NEXT: mov v5.s[0], w7
-; CHECK-GI-FP16-NEXT: ldr s6, [sp]
-; CHECK-GI-FP16-NEXT: mov v7.s[0], w4
+; CHECK-GI-FP16-NEXT: mov w10, #31 // =0x1f
+; CHECK-GI-FP16-NEXT: fmov s5, w0
+; CHECK-GI-FP16-NEXT: fmov s2, w10
+; CHECK-GI-FP16-NEXT: fmov s6, w7
+; CHECK-GI-FP16-NEXT: ldr s3, [sp]
+; CHECK-GI-FP16-NEXT: fmov s17, w4
+; CHECK-GI-FP16-NEXT: ldr s7, [sp, #24]
; CHECK-GI-FP16-NEXT: ldr s16, [sp, #32]
-; CHECK-GI-FP16-NEXT: ldr s17, [sp, #8]
+; CHECK-GI-FP16-NEXT: mov v5.s[1], w1
; CHECK-GI-FP16-NEXT: umov w8, v0.h[4]
-; CHECK-GI-FP16-NEXT: umov w10, v0.h[5]
-; CHECK-GI-FP16-NEXT: mov v4.s[1], w1
-; CHECK-GI-FP16-NEXT: mov v2.s[1], w9
-; CHECK-GI-FP16-NEXT: mov v5.s[1], v6.s[0]
-; CHECK-GI-FP16-NEXT: ldr s6, [sp, #24]
-; CHECK-GI-FP16-NEXT: mov v7.s[1], w5
-; CHECK-GI-FP16-NEXT: mov v6.s[1], v16.s[0]
-; CHECK-GI-FP16-NEXT: ldr s16, [sp, #40]
-; CHECK-GI-FP16-NEXT: mov v1.s[0], w8
+; CHECK-GI-FP16-NEXT: umov w9, v0.h[5]
+; CHECK-GI-FP16-NEXT: mov v2.s[1], w10
+; CHECK-GI-FP16-NEXT: mov v6.s[1], v3.s[0]
+; CHECK-GI-FP16-NEXT: ldr s3, [sp, #8]
+; CHECK-GI-FP16-NEXT: mov v17.s[1], w5
+; CHECK-GI-FP16-NEXT: mov v7.s[1], v16.s[0]
+; CHECK-GI-FP16-NEXT: mov v5.s[2], w2
+; CHECK-GI-FP16-NEXT: fmov s1, w8
; CHECK-GI-FP16-NEXT: umov w8, v0.h[6]
+; CHECK-GI-FP16-NEXT: mov v2.s[2], w10
; CHECK-GI-FP16-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT: mov v2.s[2], w9
-; CHECK-GI-FP16-NEXT: mov v4.s[2], w2
-; CHECK-GI-FP16-NEXT: mov v5.s[2], v17.s[0]
-; CHECK-GI-FP16-NEXT: mov v7.s[2], w6
+; CHECK-GI-FP16-NEXT: mov v6.s[2], v3.s[0]
+; CHECK-GI-FP16-NEXT: ldr s3, [sp, #40]
+; CHECK-GI-FP16-NEXT: mov v17.s[2], w6
+; CHECK-GI-FP16-NEXT: mov v1.s[1], w9
+; CHECK-GI-FP16-NEXT: mov v7.s[2], v3.s[0]
+; CHECK-GI-FP16-NEXT: mov v5.s[3], w3
; CHECK-GI-FP16-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-GI-FP16-NEXT: mov v6.s[2], v16.s[0]
-; CHECK-GI-FP16-NEXT: mov v1.s[1], w10
-; CHECK-GI-FP16-NEXT: mov w10, #-1 // =0xffffffff
-; CHECK-GI-FP16-NEXT: mov v3.s[0], w10
-; CHECK-GI-FP16-NEXT: mov v4.s[3], w3
-; CHECK-GI-FP16-NEXT: sshr v0.4s, v0.4s, #31
; CHECK-GI-FP16-NEXT: mov v1.s[2], w8
-; CHECK-GI-FP16-NEXT: mov v3.s[1], w10
+; CHECK-GI-FP16-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-FP16-NEXT: sshr v0.4s, v0.4s, #31
+; CHECK-GI-FP16-NEXT: fmov s4, w8
+; CHECK-GI-FP16-NEXT: mov v4.s[1], w8
; CHECK-GI-FP16-NEXT: ushl v1.4s, v1.4s, v2.4s
; CHECK-GI-FP16-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-FP16-NEXT: mov v3.s[2], w10
; CHECK-GI-FP16-NEXT: sshl v1.4s, v1.4s, v2.4s
; CHECK-GI-FP16-NEXT: ldr s2, [sp, #16]
-; CHECK-GI-FP16-NEXT: mov v5.s[3], v2.s[0]
-; CHECK-GI-FP16-NEXT: eor v3.16b, v1.16b, v3.16b
-; CHECK-GI-FP16-NEXT: and v1.16b, v7.16b, v1.16b
-; CHECK-GI-FP16-NEXT: and v2.16b, v6.16b, v3.16b
-; CHECK-GI-FP16-NEXT: bsl v0.16b, v4.16b, v5.16b
-; CHECK-GI-FP16-NEXT: orr v1.16b, v1.16b, v2.16b
-; CHECK-GI-FP16-NEXT: mov s2, v0.s[1]
+; CHECK-GI-FP16-NEXT: mov v4.s[2], w8
+; CHECK-GI-FP16-NEXT: mov v6.s[3], v2.s[0]
+; CHECK-GI-FP16-NEXT: eor v3.16b, v1.16b, v4.16b
+; CHECK-GI-FP16-NEXT: and v1.16b, v17.16b, v1.16b
+; CHECK-GI-FP16-NEXT: bsl v0.16b, v5.16b, v6.16b
+; CHECK-GI-FP16-NEXT: and v2.16b, v7.16b, v3.16b
; CHECK-GI-FP16-NEXT: mov s3, v0.s[2]
; CHECK-GI-FP16-NEXT: mov s4, v0.s[3]
; CHECK-GI-FP16-NEXT: fmov w0, s0
+; CHECK-GI-FP16-NEXT: orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-FP16-NEXT: mov s2, v0.s[1]
; CHECK-GI-FP16-NEXT: mov s5, v1.s[1]
; CHECK-GI-FP16-NEXT: mov s6, v1.s[2]
-; CHECK-GI-FP16-NEXT: fmov w4, s1
-; CHECK-GI-FP16-NEXT: fmov w1, s2
; CHECK-GI-FP16-NEXT: fmov w2, s3
+; CHECK-GI-FP16-NEXT: fmov w1, s2
; CHECK-GI-FP16-NEXT: fmov w3, s4
+; CHECK-GI-FP16-NEXT: fmov w4, s1
; CHECK-GI-FP16-NEXT: fmov w5, s5
; CHECK-GI-FP16-NEXT: fmov w6, s6
; CHECK-GI-FP16-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/fcopysign.ll b/llvm/test/CodeGen/AArch64/fcopysign.ll
index a42ec8e253be2..3a5f7e2cd6b29 100644
--- a/llvm/test/CodeGen/AArch64/fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/fcopysign.ll
@@ -156,8 +156,8 @@ define <3 x float> @copysign_v3f32(<3 x float> %a, <3 x float> %b) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #-2147483648 // =0x80000000
; CHECK-GI-NEXT: mov w9, #2147483647 // =0x7fffffff
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: mov v3.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w9
+; CHECK-GI-NEXT: fmov s3, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mov v3.s[1], w8
; CHECK-GI-NEXT: mov v2.s[2], w9
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
index ae3b6a54a1f7f..6a792958ee6be 100644
--- a/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -7225,7 +7225,7 @@ define <2 x i64> @fptos_v2f128_v2i64(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov x19, x0
; CHECK-GI-NEXT: bl __fixtfdi
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.d[1], x0
; CHECK-GI-NEXT: add sp, sp, #32
@@ -7268,7 +7268,7 @@ define <2 x i64> @fptou_v2f128_v2i64(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov x19, x0
; CHECK-GI-NEXT: bl __fixunstfdi
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.d[1], x0
; CHECK-GI-NEXT: add sp, sp, #32
@@ -7424,7 +7424,7 @@ define <2 x i32> @fptos_v2f128_v2i32(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixtfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -7467,7 +7467,7 @@ define <2 x i32> @fptou_v2f128_v2i32(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -7519,7 +7519,7 @@ define <3 x i32> @fptos_v3f128_v3i32(<3 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w20, w0
; CHECK-GI-NEXT: bl __fixtfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w20
; CHECK-GI-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload
@@ -7572,7 +7572,7 @@ define <3 x i32> @fptou_v3f128_v3i32(<3 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w20, w0
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w20
; CHECK-GI-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload
@@ -7616,7 +7616,7 @@ define <2 x i16> @fptos_v2f128_v2i16(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixtfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -7659,7 +7659,7 @@ define <2 x i16> @fptou_v2f128_v2i16(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -7816,7 +7816,7 @@ define <2 x i8> @fptos_v2f128_v2i8(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixtfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -7859,7 +7859,7 @@ define <2 x i8> @fptou_v2f128_v2i8(<2 x fp128> %a) {
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w0
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index 04dfdedb42752..9c21d2bf083a2 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -32,8 +32,7 @@ define <1 x i32> @test_signed_v1f32_v1i32(<1 x float> %f) {
; CHECK-GI-LABEL: test_signed_v1f32_v1i32:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: fcvtzs w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%x = call <1 x i32> @llvm.fptosi.sat.v1f32.v1i32(<1 x float> %f)
ret <1 x i32> %x
@@ -244,18 +243,11 @@ declare <5 x i32> @llvm.fptosi.sat.v5f64.v5i32 (<5 x double>)
declare <6 x i32> @llvm.fptosi.sat.v6f64.v6i32 (<6 x double>)
define <1 x i32> @test_signed_v1f64_v1i32(<1 x double> %f) {
-; CHECK-SD-LABEL: test_signed_v1f64_v1i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fcvtzs w8, d0
-; CHECK-SD-NEXT: fmov s0, w8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_signed_v1f64_v1i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtzs w8, d0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: test_signed_v1f64_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fcvtzs w8, d0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: ret
%x = call <1 x i32> @llvm.fptosi.sat.v1f64.v1i32(<1 x double> %f)
ret <1 x i32> %x
}
@@ -565,8 +557,7 @@ define <1 x i32> @test_signed_v1f128_v1i32(<1 x fp128> %f) {
; CHECK-GI-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload
; CHECK-GI-NEXT: csel w8, wzr, w19, ne
; CHECK-GI-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: ret
%x = call <1 x i32> @llvm.fptosi.sat.v1f128.v1i32(<1 x fp128> %f)
@@ -708,7 +699,7 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: mov v1.16b, v0.16b
; CHECK-GI-NEXT: bl __unordtf2
-; CHECK-GI-NEXT: mov v0.s[0], w21
+; CHECK-GI-NEXT: fmov s0, w21
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-GI-NEXT: csel w8, wzr, w19, ne
@@ -901,7 +892,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: mov v1.16b, v0.16b
; CHECK-GI-NEXT: bl __unordtf2
-; CHECK-GI-NEXT: mov v0.s[0], w21
+; CHECK-GI-NEXT: fmov s0, w21
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: csel w8, wzr, w19, ne
; CHECK-GI-NEXT: ldp x20, x19, [sp, #112] // 16-byte Folded Reload
@@ -1141,7 +1132,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
; CHECK-GI-NEXT: mov w19, w0
; CHECK-GI-NEXT: mov v1.16b, v0.16b
; CHECK-GI-NEXT: bl __unordtf2
-; CHECK-GI-NEXT: mov v0.s[0], w21
+; CHECK-GI-NEXT: fmov s0, w21
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: ldr x30, [sp, #96] // 8-byte Folded Reload
; CHECK-GI-NEXT: csel w8, wzr, w19, ne
@@ -1188,15 +1179,13 @@ define <1 x i32> @test_signed_v1f16_v1i32(<1 x half> %f) {
; CHECK-GI-CVT: // %bb.0:
; CHECK-GI-CVT-NEXT: fcvt s0, h0
; CHECK-GI-CVT-NEXT: fcvtzs w8, s0
-; CHECK-GI-CVT-NEXT: mov v0.s[0], w8
-; CHECK-GI-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-CVT-NEXT: fmov s0, w8
; CHECK-GI-CVT-NEXT: ret
;
; CHECK-GI-FP16-LABEL: test_signed_v1f16_v1i32:
; CHECK-GI-FP16: // %bb.0:
; CHECK-GI-FP16-NEXT: fcvtzs w8, h0
-; CHECK-GI-FP16-NEXT: mov v0.s[0], w8
-; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-FP16-NEXT: fmov s0, w8
; CHECK-GI-FP16-NEXT: ret
%x = call <1 x i32> @llvm.fptosi.sat.v1f16.v1i32(<1 x half> %f)
ret <1 x i32> %x
@@ -5545,7 +5534,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
; CHECK-GI-NEXT: mov x19, x0
; CHECK-GI-NEXT: mov v1.16b, v0.16b
; CHECK-GI-NEXT: bl __unordtf2
-; CHECK-GI-NEXT: mov v0.d[0], x21
+; CHECK-GI-NEXT: fmov d0, x21
; CHECK-GI-NEXT: cmp w0, #0
; CHECK-GI-NEXT: csel x8, xzr, x19, ne
; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 099f43edfca6e..44847a41287d6 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -32,8 +32,7 @@ define <1 x i32> @test_unsigned_v1f32_v1i32(<1 x float> %f) {
; CHECK-GI-LABEL: test_unsigned_v1f32_v1i32:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: fcvtzu w8, s0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%x = call <1 x i32> @llvm.fptoui.sat.v1f32.v1i32(<1 x float> %f)
ret <1 x i32> %x
@@ -244,18 +243,11 @@ declare <5 x i32> @llvm.fptoui.sat.v5f64.v5i32 (<5 x double>)
declare <6 x i32> @llvm.fptoui.sat.v6f64.v6i32 (<6 x double>)
define <1 x i32> @test_unsigned_v1f64_v1i32(<1 x double> %f) {
-; CHECK-SD-LABEL: test_unsigned_v1f64_v1i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fcvtzu w8, d0
-; CHECK-SD-NEXT: fmov s0, w8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_unsigned_v1f64_v1i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtzu w8, d0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: test_unsigned_v1f64_v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fcvtzu w8, d0
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: ret
%x = call <1 x i32> @llvm.fptoui.sat.v1f64.v1i32(<1 x double> %f)
ret <1 x i32> %x
}
@@ -513,9 +505,8 @@ define <1 x i32> @test_unsigned_v1f128_v1i32(<1 x fp128> %f) {
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: bl __fixunstfsi
; CHECK-GI-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.s[0], w0
+; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: ret
%x = call <1 x i32> @llvm.fptoui.sat.v1f128.v1i32(<1 x fp128> %f)
@@ -629,7 +620,7 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
; CHECK-GI-NEXT: csel x8, x22, x21, lt
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr x30, [sp, #48] // 8-byte Folded Reload
@@ -780,7 +771,7 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
; CHECK-GI-NEXT: csel x8, x23, x21, lt
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp x30, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w20
@@ -974,7 +965,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
; CHECK-GI-NEXT: csel x8, x24, x22, lt
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: bl __fixunstfsi
-; CHECK-GI-NEXT: mov v0.s[0], w19
+; CHECK-GI-NEXT: fmov s0, w19
; CHECK-GI-NEXT: ldp x24, x23, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldr x30, [sp, #80] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v0.s[1], w20
@@ -1019,15 +1010,13 @@ define <1 x i32> @test_unsigned_v1f16_v1i32(<1 x half> %f) {
; CHECK-GI-CVT: // %bb.0:
; CHECK-GI-CVT-NEXT: fcvt s0, h0
; CHECK-GI-CVT-NEXT: fcvtzu w8, s0
-; CHECK-GI-CVT-NEXT: mov v0.s[0], w8
-; CHECK-GI-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-CVT-NEXT: fmov s0, w8
; CHECK-GI-CVT-NEXT: ret
;
; CHECK-GI-FP16-LABEL: test_unsigned_v1f16_v1i32:
; CHECK-GI-FP16: // %bb.0:
; CHECK-GI-FP16-NEXT: fcvtzu w8, h0
-; CHECK-GI-FP16-NEXT: mov v0.s[0], w8
-; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-FP16-NEXT: fmov s0, w8
; CHECK-GI-FP16-NEXT: ret
%x = call <1 x i32> @llvm.fptoui.sat.v1f16.v1i32(<1 x half> %f)
ret <1 x i32> %x
@@ -4515,7 +4504,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
; CHECK-GI-NEXT: csel x8, x23, x22, lt
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: bl __fixunstfdi
-; CHECK-GI-NEXT: mov v0.d[0], x19
+; CHECK-GI-NEXT: fmov d0, x19
; CHECK-GI-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ldp x30, x23, [sp, #48] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index 6efd9f40f0068..0c56e1b66e81f 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -137,10 +137,9 @@ define <2 x i16> @freeze_v2i16() {
; CHECK-GI-LABEL: freeze_v2i16:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov h0, v0.h[1]
-; CHECK-GI-NEXT: mov v1.s[0], w8
; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v1.s[1], w8
-; CHECK-GI-NEXT: add v0.2s, v1.2s, v1.2s
+; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: add v0.2s, v0.2s, v0.2s
; CHECK-GI-NEXT: ret
%y1 = freeze <2 x i16> undef
%t1 = add <2 x i16> %y1, %y1
diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll
index e29f35cfc2173..4c28c90824028 100644
--- a/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/llvm/test/CodeGen/AArch64/fsh.ll
@@ -1378,77 +1378,77 @@ define <7 x i32> @rotl_v7i32(<7 x i32> %a, <7 x i32> %c) {
;
; CHECK-GI-LABEL: rotl_v7i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v1.s[0], w7
; CHECK-GI-NEXT: ldr s0, [sp, #24]
-; CHECK-GI-NEXT: mov v2.s[0], wzr
-; CHECK-GI-NEXT: mov v3.s[0], w7
-; CHECK-GI-NEXT: mov x9, sp
-; CHECK-GI-NEXT: ldr s5, [sp, #32]
-; CHECK-GI-NEXT: mov v4.16b, v0.16b
-; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s3, w7
+; CHECK-GI-NEXT: ldr s2, [sp, #32]
+; CHECK-GI-NEXT: mov x8, sp
+; CHECK-GI-NEXT: fmov s4, w7
+; CHECK-GI-NEXT: mov v6.16b, v0.16b
; CHECK-GI-NEXT: ldr s7, [sp]
-; CHECK-GI-NEXT: mov v16.s[0], w8
-; CHECK-GI-NEXT: mov v6.s[0], w0
-; CHECK-GI-NEXT: ldr s18, [sp, #40]
-; CHECK-GI-NEXT: ld1 { v1.s }[1], [x9]
-; CHECK-GI-NEXT: mov v2.s[1], wzr
-; CHECK-GI-NEXT: add x9, sp, #8
-; CHECK-GI-NEXT: mov v4.s[1], v5.s[0]
-; CHECK-GI-NEXT: mov v3.s[1], v7.s[0]
-; CHECK-GI-NEXT: mov v7.s[0], w0
-; CHECK-GI-NEXT: mov v19.s[0], w8
-; CHECK-GI-NEXT: ldr s17, [sp, #8]
-; CHECK-GI-NEXT: mov v20.s[0], w4
-; CHECK-GI-NEXT: ld1 { v1.s }[2], [x9]
-; CHECK-GI-NEXT: mov v16.s[1], w8
+; CHECK-GI-NEXT: ldr s5, [sp, #40]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: ld1 { v3.s }[1], [x8]
+; CHECK-GI-NEXT: add x8, sp, #8
+; CHECK-GI-NEXT: mov v4.s[1], v7.s[0]
+; CHECK-GI-NEXT: ldr s7, [sp, #8]
+; CHECK-GI-NEXT: fmov s16, w0
+; CHECK-GI-NEXT: mov v6.s[1], v2.s[0]
+; CHECK-GI-NEXT: fmov s17, w0
; CHECK-GI-NEXT: add x9, sp, #16
-; CHECK-GI-NEXT: mov v2.s[2], wzr
-; CHECK-GI-NEXT: mov v6.s[1], w1
-; CHECK-GI-NEXT: mov v21.s[0], w4
-; CHECK-GI-NEXT: mov v4.s[2], v18.s[0]
-; CHECK-GI-NEXT: mov v7.s[1], w1
-; CHECK-GI-NEXT: mov v3.s[2], v17.s[0]
-; CHECK-GI-NEXT: ld1 { v1.s }[3], [x9]
-; CHECK-GI-NEXT: mov v0.s[1], v5.s[0]
-; CHECK-GI-NEXT: mov v19.s[1], w8
-; CHECK-GI-NEXT: movi v5.4s, #31
-; CHECK-GI-NEXT: mov v16.s[2], w8
-; CHECK-GI-NEXT: ldr s17, [sp, #16]
-; CHECK-GI-NEXT: mov v6.s[2], w2
-; CHECK-GI-NEXT: mov v20.s[1], w5
-; CHECK-GI-NEXT: mov v21.s[1], w5
-; CHECK-GI-NEXT: sub v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT: neg v1.4s, v1.4s
-; CHECK-GI-NEXT: mov v7.s[2], w2
-; CHECK-GI-NEXT: mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT: mov v0.s[2], v18.s[0]
-; CHECK-GI-NEXT: mov v19.s[2], w8
-; CHECK-GI-NEXT: and v1.16b, v1.16b, v5.16b
-; CHECK-GI-NEXT: and v2.16b, v2.16b, v16.16b
-; CHECK-GI-NEXT: mov v6.s[3], w3
-; CHECK-GI-NEXT: mov v7.s[3], w3
-; CHECK-GI-NEXT: mov v20.s[2], w6
-; CHECK-GI-NEXT: mov v21.s[2], w6
-; CHECK-GI-NEXT: and v3.16b, v3.16b, v5.16b
-; CHECK-GI-NEXT: and v0.16b, v0.16b, v19.16b
+; CHECK-GI-NEXT: ld1 { v3.s }[2], [x8]
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: fmov s18, w8
+; CHECK-GI-NEXT: mov v16.s[1], w1
+; CHECK-GI-NEXT: mov v4.s[2], v7.s[0]
+; CHECK-GI-NEXT: ldr s7, [sp, #16]
+; CHECK-GI-NEXT: mov v17.s[1], w1
+; CHECK-GI-NEXT: mov v6.s[2], v5.s[0]
+; CHECK-GI-NEXT: ld1 { v3.s }[3], [x9]
+; CHECK-GI-NEXT: fmov s2, w4
+; CHECK-GI-NEXT: mov v18.s[1], w8
+; CHECK-GI-NEXT: movi v19.4s, #31
+; CHECK-GI-NEXT: mov v0.s[2], v5.s[0]
+; CHECK-GI-NEXT: mov v16.s[2], w2
+; CHECK-GI-NEXT: mov v4.s[3], v7.s[0]
+; CHECK-GI-NEXT: fmov s7, w4
+; CHECK-GI-NEXT: neg v3.4s, v3.4s
+; CHECK-GI-NEXT: sub v1.4s, v1.4s, v6.4s
+; CHECK-GI-NEXT: fmov s6, w8
+; CHECK-GI-NEXT: mov v17.s[2], w2
+; CHECK-GI-NEXT: mov v18.s[2], w8
+; CHECK-GI-NEXT: mov v2.s[1], w5
+; CHECK-GI-NEXT: mov v7.s[1], w5
+; CHECK-GI-NEXT: and v3.16b, v3.16b, v19.16b
+; CHECK-GI-NEXT: mov v16.s[3], w3
+; CHECK-GI-NEXT: mov v6.s[1], w8
+; CHECK-GI-NEXT: and v4.16b, v4.16b, v19.16b
+; CHECK-GI-NEXT: mov v17.s[3], w3
+; CHECK-GI-NEXT: and v1.16b, v1.16b, v18.16b
+; CHECK-GI-NEXT: mov v2.s[2], w6
+; CHECK-GI-NEXT: neg v3.4s, v3.4s
+; CHECK-GI-NEXT: mov v7.s[2], w6
+; CHECK-GI-NEXT: mov v6.s[2], w8
; CHECK-GI-NEXT: neg v1.4s, v1.4s
-; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT: ushl v0.4s, v20.4s, v0.4s
-; CHECK-GI-NEXT: ushl v1.4s, v7.4s, v1.4s
-; CHECK-GI-NEXT: ushl v2.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: mov s2, v1.s[1]
-; CHECK-GI-NEXT: mov s3, v1.s[2]
-; CHECK-GI-NEXT: mov s4, v1.s[3]
+; CHECK-GI-NEXT: ushl v4.4s, v17.4s, v4.4s
+; CHECK-GI-NEXT: ushl v3.4s, v16.4s, v3.4s
+; CHECK-GI-NEXT: and v0.16b, v0.16b, v6.16b
+; CHECK-GI-NEXT: ushl v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: orr v2.16b, v4.16b, v3.16b
+; CHECK-GI-NEXT: ushl v0.4s, v7.4s, v0.4s
+; CHECK-GI-NEXT: mov s3, v2.s[2]
+; CHECK-GI-NEXT: mov s4, v2.s[3]
+; CHECK-GI-NEXT: fmov w0, s2
+; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: mov s1, v2.s[1]
+; CHECK-GI-NEXT: fmov w2, s3
+; CHECK-GI-NEXT: fmov w3, s4
; CHECK-GI-NEXT: mov s5, v0.s[1]
; CHECK-GI-NEXT: mov s6, v0.s[2]
-; CHECK-GI-NEXT: fmov w0, s1
; CHECK-GI-NEXT: fmov w4, s0
-; CHECK-GI-NEXT: fmov w1, s2
-; CHECK-GI-NEXT: fmov w2, s3
-; CHECK-GI-NEXT: fmov w3, s4
+; CHECK-GI-NEXT: fmov w1, s1
; CHECK-GI-NEXT: fmov w5, s5
; CHECK-GI-NEXT: fmov w6, s6
; CHECK-GI-NEXT: ret
@@ -1505,74 +1505,74 @@ define <7 x i32> @rotr_v7i32(<7 x i32> %a, <7 x i32> %c) {
;
; CHECK-GI-LABEL: rotr_v7i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w7
-; CHECK-GI-NEXT: ldr s2, [sp]
-; CHECK-GI-NEXT: mov v1.s[0], w7
+; CHECK-GI-NEXT: fmov s1, w7
+; CHECK-GI-NEXT: ldr s3, [sp]
+; CHECK-GI-NEXT: fmov s2, w7
+; CHECK-GI-NEXT: mov x8, sp
+; CHECK-GI-NEXT: ldr s6, [sp, #8]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: ldr s7, [sp, #32]
+; CHECK-GI-NEXT: fmov s16, w0
+; CHECK-GI-NEXT: fmov s17, w0
+; CHECK-GI-NEXT: mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT: ldr s3, [sp, #24]
+; CHECK-GI-NEXT: ld1 { v2.s }[1], [x8]
; CHECK-GI-NEXT: mov w8, #31 // =0x1f
-; CHECK-GI-NEXT: ldr s7, [sp, #8]
-; CHECK-GI-NEXT: mov v3.s[0], wzr
-; CHECK-GI-NEXT: mov v5.s[0], w8
-; CHECK-GI-NEXT: mov x11, sp
-; CHECK-GI-NEXT: ldr s16, [sp, #32]
-; CHECK-GI-NEXT: mov v4.s[0], w0
-; CHECK-GI-NEXT: mov v17.s[0], w0
-; CHECK-GI-NEXT: ldr s6, [sp, #16]
-; CHECK-GI-NEXT: mov v0.s[1], v2.s[0]
-; CHECK-GI-NEXT: ldr s2, [sp, #24]
-; CHECK-GI-NEXT: ld1 { v1.s }[1], [x11]
-; CHECK-GI-NEXT: mov v3.s[1], wzr
-; CHECK-GI-NEXT: add x10, sp, #8
-; CHECK-GI-NEXT: ldr s18, [sp, #40]
-; CHECK-GI-NEXT: mov v19.16b, v2.16b
-; CHECK-GI-NEXT: mov v2.s[1], v16.s[0]
-; CHECK-GI-NEXT: mov v5.s[1], w8
-; CHECK-GI-NEXT: ld1 { v1.s }[2], [x10]
-; CHECK-GI-NEXT: mov v4.s[1], w1
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: add x9, sp, #8
+; CHECK-GI-NEXT: mov v4.16b, v3.16b
+; CHECK-GI-NEXT: mov v3.s[1], v7.s[0]
+; CHECK-GI-NEXT: fmov s18, w8
+; CHECK-GI-NEXT: ldr s5, [sp, #40]
+; CHECK-GI-NEXT: ld1 { v2.s }[2], [x9]
; CHECK-GI-NEXT: mov v17.s[1], w1
-; CHECK-GI-NEXT: mov v0.s[2], v7.s[0]
-; CHECK-GI-NEXT: mov v7.s[0], w8
-; CHECK-GI-NEXT: mov v20.s[0], w4
-; CHECK-GI-NEXT: mov v19.s[1], v16.s[0]
-; CHECK-GI-NEXT: add x9, sp, #16
-; CHECK-GI-NEXT: movi v16.4s, #31
-; CHECK-GI-NEXT: mov v2.s[2], v18.s[0]
-; CHECK-GI-NEXT: mov v3.s[2], wzr
-; CHECK-GI-NEXT: mov v5.s[2], w8
-; CHECK-GI-NEXT: ld1 { v1.s }[3], [x9]
-; CHECK-GI-NEXT: mov v4.s[2], w2
+; CHECK-GI-NEXT: mov v1.s[2], v6.s[0]
+; CHECK-GI-NEXT: fmov s6, w8
+; CHECK-GI-NEXT: mov v16.s[1], w1
+; CHECK-GI-NEXT: mov v4.s[1], v7.s[0]
+; CHECK-GI-NEXT: ldr s7, [sp, #16]
+; CHECK-GI-NEXT: mov v18.s[1], w8
+; CHECK-GI-NEXT: mov v3.s[2], v5.s[0]
+; CHECK-GI-NEXT: fmov s19, w4
+; CHECK-GI-NEXT: add x10, sp, #16
+; CHECK-GI-NEXT: mov v6.s[1], w8
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: ld1 { v2.s }[3], [x10]
+; CHECK-GI-NEXT: mov v1.s[3], v7.s[0]
+; CHECK-GI-NEXT: movi v7.4s, #31
; CHECK-GI-NEXT: mov v17.s[2], w2
-; CHECK-GI-NEXT: mov v0.s[3], v6.s[0]
-; CHECK-GI-NEXT: mov v6.s[0], w4
-; CHECK-GI-NEXT: mov v7.s[1], w8
-; CHECK-GI-NEXT: mov v19.s[2], v18.s[0]
-; CHECK-GI-NEXT: mov v20.s[1], w5
-; CHECK-GI-NEXT: neg v1.4s, v1.4s
-; CHECK-GI-NEXT: and v2.16b, v2.16b, v5.16b
-; CHECK-GI-NEXT: mov v4.s[3], w3
-; CHECK-GI-NEXT: mov v17.s[3], w3
-; CHECK-GI-NEXT: mov v6.s[1], w5
-; CHECK-GI-NEXT: mov v7.s[2], w8
-; CHECK-GI-NEXT: and v0.16b, v0.16b, v16.16b
-; CHECK-GI-NEXT: sub v3.4s, v3.4s, v19.4s
-; CHECK-GI-NEXT: mov v20.s[2], w6
-; CHECK-GI-NEXT: and v1.16b, v1.16b, v16.16b
+; CHECK-GI-NEXT: mov v4.s[2], v5.s[0]
+; CHECK-GI-NEXT: fmov s5, w4
+; CHECK-GI-NEXT: mov v16.s[2], w2
+; CHECK-GI-NEXT: mov v19.s[1], w5
+; CHECK-GI-NEXT: mov v18.s[2], w8
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: neg v0.4s, v0.4s
-; CHECK-GI-NEXT: mov v6.s[2], w6
-; CHECK-GI-NEXT: and v3.16b, v3.16b, v7.16b
+; CHECK-GI-NEXT: mov v6.s[2], w8
+; CHECK-GI-NEXT: mov v5.s[1], w5
+; CHECK-GI-NEXT: and v1.16b, v1.16b, v7.16b
+; CHECK-GI-NEXT: mov v17.s[3], w3
+; CHECK-GI-NEXT: sub v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: mov v16.s[3], w3
+; CHECK-GI-NEXT: and v2.16b, v2.16b, v7.16b
+; CHECK-GI-NEXT: mov v19.s[2], w6
+; CHECK-GI-NEXT: and v3.16b, v3.16b, v6.16b
+; CHECK-GI-NEXT: neg v1.4s, v1.4s
+; CHECK-GI-NEXT: mov v5.s[2], w6
+; CHECK-GI-NEXT: and v0.16b, v0.16b, v18.16b
+; CHECK-GI-NEXT: ushl v2.4s, v16.4s, v2.4s
+; CHECK-GI-NEXT: neg v3.4s, v3.4s
; CHECK-GI-NEXT: ushl v1.4s, v17.4s, v1.4s
-; CHECK-GI-NEXT: ushl v2.4s, v20.4s, v2.4s
-; CHECK-GI-NEXT: ushl v0.4s, v4.4s, v0.4s
-; CHECK-GI-NEXT: ushl v3.4s, v6.4s, v3.4s
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: orr v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT: mov s2, v0.s[1]
-; CHECK-GI-NEXT: mov s3, v0.s[2]
-; CHECK-GI-NEXT: mov s4, v0.s[3]
-; CHECK-GI-NEXT: fmov w0, s0
-; CHECK-GI-NEXT: mov s5, v1.s[1]
-; CHECK-GI-NEXT: mov s6, v1.s[2]
-; CHECK-GI-NEXT: fmov w4, s1
+; CHECK-GI-NEXT: ushl v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT: ushl v3.4s, v19.4s, v3.4s
+; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT: orr v0.16b, v3.16b, v0.16b
+; CHECK-GI-NEXT: mov s2, v1.s[1]
+; CHECK-GI-NEXT: mov s3, v1.s[2]
+; CHECK-GI-NEXT: mov s4, v1.s[3]
+; CHECK-GI-NEXT: fmov w0, s1
+; CHECK-GI-NEXT: mov s5, v0.s[1]
+; CHECK-GI-NEXT: mov s6, v0.s[2]
+; CHECK-GI-NEXT: fmov w4, s0
; CHECK-GI-NEXT: fmov w1, s2
; CHECK-GI-NEXT: fmov w2, s3
; CHECK-GI-NEXT: fmov w3, s4
@@ -2509,78 +2509,78 @@ define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
;
; CHECK-GI-LABEL: fshl_v7i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov w11, #-1 // =0xffffffff
; CHECK-GI-NEXT: ldr s3, [sp, #48]
-; CHECK-GI-NEXT: ldr s19, [sp, #56]
-; CHECK-GI-NEXT: mov v20.s[0], w11
-; CHECK-GI-NEXT: mov v21.s[0], w7
-; CHECK-GI-NEXT: ldr s16, [sp, #80]
-; CHECK-GI-NEXT: ldr s22, [sp, #88]
-; CHECK-GI-NEXT: mov w12, #31 // =0x1f
+; CHECK-GI-NEXT: ldr s20, [sp, #56]
+; CHECK-GI-NEXT: add x9, sp, #56
+; CHECK-GI-NEXT: ldr s4, [sp, #48]
+; CHECK-GI-NEXT: ldr s7, [sp, #80]
+; CHECK-GI-NEXT: mov w12, #-1 // =0xffffffff
+; CHECK-GI-NEXT: ldr s21, [sp, #88]
+; CHECK-GI-NEXT: mov v3.s[1], v20.s[0]
+; CHECK-GI-NEXT: fmov s20, w12
+; CHECK-GI-NEXT: ld1 { v4.s }[1], [x9]
+; CHECK-GI-NEXT: ldr s17, [sp]
+; CHECK-GI-NEXT: add x13, sp, #64
+; CHECK-GI-NEXT: mov v7.s[1], v21.s[0]
+; CHECK-GI-NEXT: fmov s21, w7
+; CHECK-GI-NEXT: ldr s19, [sp, #64]
+; CHECK-GI-NEXT: mov w11, #31 // =0x1f
+; CHECK-GI-NEXT: mov v20.s[1], w12
+; CHECK-GI-NEXT: ldr s18, [sp, #96]
+; CHECK-GI-NEXT: ld1 { v4.s }[2], [x13]
; CHECK-GI-NEXT: mov w13, #1 // =0x1
-; CHECK-GI-NEXT: ldr s6, [sp]
-; CHECK-GI-NEXT: mov v3.s[1], v19.s[0]
-; CHECK-GI-NEXT: mov v19.s[0], w12
-; CHECK-GI-NEXT: mov v23.s[0], w13
-; CHECK-GI-NEXT: mov v16.s[1], v22.s[0]
-; CHECK-GI-NEXT: ldr s7, [sp, #64]
-; CHECK-GI-NEXT: mov v20.s[1], w11
-; CHECK-GI-NEXT: mov v22.s[0], w0
-; CHECK-GI-NEXT: mov v21.s[1], v6.s[0]
-; CHECK-GI-NEXT: ldr s24, [sp, #96]
-; CHECK-GI-NEXT: mov v6.s[0], w12
-; CHECK-GI-NEXT: ldr s5, [sp, #8]
-; CHECK-GI-NEXT: ldr s18, [sp, #48]
-; CHECK-GI-NEXT: mov v3.s[2], v7.s[0]
-; CHECK-GI-NEXT: mov v19.s[1], w12
-; CHECK-GI-NEXT: mov v23.s[1], w13
+; CHECK-GI-NEXT: mov v3.s[2], v19.s[0]
+; CHECK-GI-NEXT: mov v21.s[1], v17.s[0]
+; CHECK-GI-NEXT: fmov s17, w11
+; CHECK-GI-NEXT: fmov s19, w13
+; CHECK-GI-NEXT: fmov s23, w0
+; CHECK-GI-NEXT: fmov s24, w11
+; CHECK-GI-NEXT: ldr s6, [sp, #8]
; CHECK-GI-NEXT: ldr s0, [sp, #24]
-; CHECK-GI-NEXT: ldr s4, [sp, #32]
-; CHECK-GI-NEXT: add x10, sp, #56
-; CHECK-GI-NEXT: mov v16.s[2], v24.s[0]
-; CHECK-GI-NEXT: mov v20.s[2], w11
-; CHECK-GI-NEXT: ldr s17, [sp, #72]
-; CHECK-GI-NEXT: ld1 { v18.s }[1], [x10]
-; CHECK-GI-NEXT: mov v22.s[1], w1
-; CHECK-GI-NEXT: mov v21.s[2], v5.s[0]
-; CHECK-GI-NEXT: mov v5.s[0], w4
-; CHECK-GI-NEXT: ldr s24, [sp, #80]
-; CHECK-GI-NEXT: mov v6.s[1], w12
-; CHECK-GI-NEXT: mov v0.s[1], v4.s[0]
-; CHECK-GI-NEXT: add x9, sp, #64
+; CHECK-GI-NEXT: ldr s5, [sp, #32]
+; CHECK-GI-NEXT: mov v7.s[2], v18.s[0]
+; CHECK-GI-NEXT: mov v17.s[1], w11
+; CHECK-GI-NEXT: mov v19.s[1], w13
+; CHECK-GI-NEXT: mov v20.s[2], w12
+; CHECK-GI-NEXT: ldr s16, [sp, #72]
+; CHECK-GI-NEXT: mov v23.s[1], w1
+; CHECK-GI-NEXT: ldr s18, [sp, #80]
+; CHECK-GI-NEXT: mov v21.s[2], v6.s[0]
+; CHECK-GI-NEXT: mov v24.s[1], w11
+; CHECK-GI-NEXT: mov v0.s[1], v5.s[0]
+; CHECK-GI-NEXT: fmov s6, w4
; CHECK-GI-NEXT: add x10, sp, #88
-; CHECK-GI-NEXT: movi v7.4s, #31
-; CHECK-GI-NEXT: mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT: mov v19.s[2], w12
-; CHECK-GI-NEXT: mov v23.s[2], w13
+; CHECK-GI-NEXT: movi v22.4s, #31
+; CHECK-GI-NEXT: mov v3.s[3], v16.s[0]
+; CHECK-GI-NEXT: mov v17.s[2], w11
+; CHECK-GI-NEXT: mov v19.s[2], w13
; CHECK-GI-NEXT: ldr s2, [sp, #16]
; CHECK-GI-NEXT: ldr s1, [sp, #40]
-; CHECK-GI-NEXT: ld1 { v18.s }[2], [x9]
-; CHECK-GI-NEXT: ld1 { v24.s }[1], [x10]
-; CHECK-GI-NEXT: eor v4.16b, v16.16b, v20.16b
-; CHECK-GI-NEXT: mov v22.s[2], w2
-; CHECK-GI-NEXT: mov v5.s[1], w5
+; CHECK-GI-NEXT: ld1 { v18.s }[1], [x10]
+; CHECK-GI-NEXT: eor v5.16b, v7.16b, v20.16b
+; CHECK-GI-NEXT: mov v23.s[2], w2
+; CHECK-GI-NEXT: mov v6.s[1], w5
; CHECK-GI-NEXT: add x8, sp, #72
; CHECK-GI-NEXT: add x9, sp, #96
; CHECK-GI-NEXT: mov v21.s[3], v2.s[0]
-; CHECK-GI-NEXT: mov v6.s[2], w12
+; CHECK-GI-NEXT: mov v24.s[2], w11
; CHECK-GI-NEXT: mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT: ld1 { v18.s }[3], [x8]
-; CHECK-GI-NEXT: bic v2.16b, v7.16b, v3.16b
-; CHECK-GI-NEXT: ld1 { v24.s }[2], [x9]
-; CHECK-GI-NEXT: and v1.16b, v4.16b, v19.16b
-; CHECK-GI-NEXT: neg v3.4s, v23.4s
-; CHECK-GI-NEXT: mov v22.s[3], w3
-; CHECK-GI-NEXT: mov v5.s[2], w6
-; CHECK-GI-NEXT: and v4.16b, v18.16b, v7.16b
-; CHECK-GI-NEXT: ushr v7.4s, v21.4s, #1
+; CHECK-GI-NEXT: ld1 { v4.s }[3], [x8]
+; CHECK-GI-NEXT: bic v2.16b, v22.16b, v3.16b
+; CHECK-GI-NEXT: ld1 { v18.s }[2], [x9]
+; CHECK-GI-NEXT: and v1.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT: neg v3.4s, v19.4s
+; CHECK-GI-NEXT: mov v23.s[3], w3
+; CHECK-GI-NEXT: mov v6.s[2], w6
+; CHECK-GI-NEXT: and v4.16b, v4.16b, v22.16b
+; CHECK-GI-NEXT: ushr v5.4s, v21.4s, #1
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: and v6.16b, v24.16b, v6.16b
+; CHECK-GI-NEXT: and v7.16b, v18.16b, v24.16b
; CHECK-GI-NEXT: neg v1.4s, v1.4s
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT: ushl v3.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT: ushl v2.4s, v7.4s, v2.4s
-; CHECK-GI-NEXT: ushl v4.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: ushl v3.4s, v23.4s, v4.4s
+; CHECK-GI-NEXT: ushl v2.4s, v5.4s, v2.4s
+; CHECK-GI-NEXT: ushl v4.4s, v6.4s, v7.4s
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT: orr v1.16b, v3.16b, v2.16b
; CHECK-GI-NEXT: orr v0.16b, v4.16b, v0.16b
@@ -2662,86 +2662,86 @@ define <7 x i32> @fshr_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
;
; CHECK-GI-LABEL: fshr_v7i32:
; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr s4, [sp, #48]
+; CHECK-GI-NEXT: add x8, sp, #56
; CHECK-GI-NEXT: ldr s2, [sp, #48]
-; CHECK-GI-NEXT: ldr s16, [sp, #56]
-; CHECK-GI-NEXT: add x10, sp, #56
-; CHECK-GI-NEXT: ldr s4, [sp, #80]
-; CHECK-GI-NEXT: ldr s17, [sp, #88]
-; CHECK-GI-NEXT: mov w11, #-1 // =0xffffffff
-; CHECK-GI-NEXT: ldr s7, [sp, #48]
-; CHECK-GI-NEXT: mov v2.s[1], v16.s[0]
-; CHECK-GI-NEXT: mov v16.s[0], w0
-; CHECK-GI-NEXT: mov v4.s[1], v17.s[0]
-; CHECK-GI-NEXT: ldr s19, [sp, #64]
-; CHECK-GI-NEXT: mov v18.s[0], w11
-; CHECK-GI-NEXT: ld1 { v7.s }[1], [x10]
-; CHECK-GI-NEXT: mov w10, #31 // =0x1f
-; CHECK-GI-NEXT: add x9, sp, #64
-; CHECK-GI-NEXT: mov v17.s[0], w10
+; CHECK-GI-NEXT: ldr s18, [sp, #56]
+; CHECK-GI-NEXT: ldr s5, [sp, #80]
+; CHECK-GI-NEXT: add x9, sp, #72
+; CHECK-GI-NEXT: ld1 { v4.s }[1], [x8]
+; CHECK-GI-NEXT: ldr s19, [sp, #88]
+; CHECK-GI-NEXT: add x8, sp, #64
+; CHECK-GI-NEXT: mov v2.s[1], v18.s[0]
+; CHECK-GI-NEXT: ldr s20, [sp, #64]
; CHECK-GI-NEXT: ldr s21, [sp, #96]
-; CHECK-GI-NEXT: mov v22.s[0], w4
-; CHECK-GI-NEXT: mov v2.s[2], v19.s[0]
-; CHECK-GI-NEXT: mov v19.s[0], w7
-; CHECK-GI-NEXT: mov v16.s[1], w1
-; CHECK-GI-NEXT: ld1 { v7.s }[2], [x9]
-; CHECK-GI-NEXT: mov w9, #1 // =0x1
-; CHECK-GI-NEXT: mov v4.s[2], v21.s[0]
-; CHECK-GI-NEXT: mov v21.s[0], w10
-; CHECK-GI-NEXT: mov v23.s[0], w9
-; CHECK-GI-NEXT: ldr s6, [sp]
-; CHECK-GI-NEXT: ldr s24, [sp, #80]
-; CHECK-GI-NEXT: mov v17.s[1], w10
-; CHECK-GI-NEXT: mov v18.s[1], w11
-; CHECK-GI-NEXT: add x12, sp, #88
-; CHECK-GI-NEXT: mov v19.s[1], v6.s[0]
-; CHECK-GI-NEXT: add x8, sp, #72
-; CHECK-GI-NEXT: ld1 { v24.s }[1], [x12]
-; CHECK-GI-NEXT: mov v16.s[2], w2
-; CHECK-GI-NEXT: mov v22.s[1], w5
-; CHECK-GI-NEXT: mov v21.s[1], w10
-; CHECK-GI-NEXT: mov v23.s[1], w9
-; CHECK-GI-NEXT: ldr s5, [sp, #8]
+; CHECK-GI-NEXT: mov v5.s[1], v19.s[0]
+; CHECK-GI-NEXT: fmov s19, w0
+; CHECK-GI-NEXT: ldr s7, [sp, #80]
+; CHECK-GI-NEXT: ld1 { v4.s }[2], [x8]
+; CHECK-GI-NEXT: mov w8, #31 // =0x1f
+; CHECK-GI-NEXT: ldr s16, [sp]
+; CHECK-GI-NEXT: fmov s22, w8
+; CHECK-GI-NEXT: add x10, sp, #88
+; CHECK-GI-NEXT: fmov s23, w8
+; CHECK-GI-NEXT: mov v2.s[2], v20.s[0]
+; CHECK-GI-NEXT: ld1 { v7.s }[1], [x10]
+; CHECK-GI-NEXT: mov v19.s[1], w1
+; CHECK-GI-NEXT: ld1 { v4.s }[3], [x9]
+; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT: mov v5.s[2], v21.s[0]
+; CHECK-GI-NEXT: fmov s20, w9
+; CHECK-GI-NEXT: fmov s21, w7
+; CHECK-GI-NEXT: mov w10, #1 // =0x1
+; CHECK-GI-NEXT: mov v22.s[1], w8
+; CHECK-GI-NEXT: fmov s24, w10
+; CHECK-GI-NEXT: mov v23.s[1], w8
+; CHECK-GI-NEXT: mov v19.s[2], w2
+; CHECK-GI-NEXT: ldr s6, [sp, #8]
; CHECK-GI-NEXT: ldr s0, [sp, #24]
+; CHECK-GI-NEXT: mov v20.s[1], w9
+; CHECK-GI-NEXT: mov v21.s[1], v16.s[0]
+; CHECK-GI-NEXT: fmov s16, w4
+; CHECK-GI-NEXT: mov v24.s[1], w10
; CHECK-GI-NEXT: ldr s3, [sp, #32]
-; CHECK-GI-NEXT: ld1 { v7.s }[3], [x8]
-; CHECK-GI-NEXT: movi v6.4s, #31
-; CHECK-GI-NEXT: add x8, sp, #96
-; CHECK-GI-NEXT: mov v17.s[2], w10
-; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: ldr s20, [sp, #72]
-; CHECK-GI-NEXT: ld1 { v24.s }[2], [x8]
-; CHECK-GI-NEXT: mov v19.s[2], v5.s[0]
+; CHECK-GI-NEXT: movi v18.4s, #31
+; CHECK-GI-NEXT: add x11, sp, #96
+; CHECK-GI-NEXT: mov v22.s[2], w8
+; CHECK-GI-NEXT: ldr s17, [sp, #72]
+; CHECK-GI-NEXT: mov v16.s[1], w5
+; CHECK-GI-NEXT: ld1 { v7.s }[2], [x11]
; CHECK-GI-NEXT: mov v0.s[1], v3.s[0]
-; CHECK-GI-NEXT: mov v16.s[3], w3
-; CHECK-GI-NEXT: mov v2.s[3], v20.s[0]
-; CHECK-GI-NEXT: mov v21.s[2], w10
-; CHECK-GI-NEXT: mov v22.s[2], w6
-; CHECK-GI-NEXT: mov v23.s[2], w9
+; CHECK-GI-NEXT: mov v20.s[2], w9
+; CHECK-GI-NEXT: mov v21.s[2], v6.s[0]
+; CHECK-GI-NEXT: mov v2.s[3], v17.s[0]
+; CHECK-GI-NEXT: mov v19.s[3], w3
+; CHECK-GI-NEXT: mov v23.s[2], w8
+; CHECK-GI-NEXT: mov v24.s[2], w10
; CHECK-GI-NEXT: ldr s1, [sp, #16]
-; CHECK-GI-NEXT: and v5.16b, v7.16b, v6.16b
+; CHECK-GI-NEXT: and v4.16b, v4.16b, v18.16b
; CHECK-GI-NEXT: ldr s3, [sp, #40]
-; CHECK-GI-NEXT: and v7.16b, v24.16b, v17.16b
-; CHECK-GI-NEXT: eor v4.16b, v4.16b, v18.16b
-; CHECK-GI-NEXT: mov v19.s[3], v1.s[0]
-; CHECK-GI-NEXT: shl v1.4s, v16.4s, #1
+; CHECK-GI-NEXT: mov v16.s[2], w6
+; CHECK-GI-NEXT: and v6.16b, v7.16b, v22.16b
+; CHECK-GI-NEXT: eor v5.16b, v5.16b, v20.16b
+; CHECK-GI-NEXT: mov v21.s[3], v1.s[0]
; CHECK-GI-NEXT: mov v0.s[2], v3.s[0]
-; CHECK-GI-NEXT: bic v2.16b, v6.16b, v2.16b
-; CHECK-GI-NEXT: neg v5.4s, v5.4s
-; CHECK-GI-NEXT: and v3.16b, v4.16b, v21.16b
-; CHECK-GI-NEXT: ushl v4.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT: neg v6.4s, v7.4s
+; CHECK-GI-NEXT: bic v2.16b, v18.16b, v2.16b
+; CHECK-GI-NEXT: shl v1.4s, v19.4s, #1
+; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: neg v6.4s, v6.4s
+; CHECK-GI-NEXT: and v3.16b, v5.16b, v23.16b
+; CHECK-GI-NEXT: ushl v5.4s, v16.4s, v24.4s
; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: ushl v2.4s, v19.4s, v5.4s
-; CHECK-GI-NEXT: ushl v3.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT: ushl v2.4s, v21.4s, v4.4s
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v6.4s
+; CHECK-GI-NEXT: ushl v3.4s, v5.4s, v3.4s
; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT: orr v0.16b, v3.16b, v0.16b
; CHECK-GI-NEXT: mov s2, v1.s[1]
; CHECK-GI-NEXT: mov s3, v1.s[2]
; CHECK-GI-NEXT: mov s4, v1.s[3]
+; CHECK-GI-NEXT: fmov w0, s1
; CHECK-GI-NEXT: mov s5, v0.s[1]
; CHECK-GI-NEXT: mov s6, v0.s[2]
-; CHECK-GI-NEXT: fmov w0, s1
; CHECK-GI-NEXT: fmov w4, s0
; CHECK-GI-NEXT: fmov w1, s2
; CHECK-GI-NEXT: fmov w2, s3
@@ -3587,35 +3587,35 @@ define <7 x i32> @rotl_v7i32_c(<7 x i32> %a) {
;
; CHECK-GI-LABEL: rotl_v7i32_c:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v1.s[0], w0
+; CHECK-GI-NEXT: fmov s0, w0
+; CHECK-GI-NEXT: fmov s1, w0
; CHECK-GI-NEXT: mov w8, #29 // =0x1d
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov w9, #3 // =0x3
-; CHECK-GI-NEXT: mov v3.s[0], w4
-; CHECK-GI-NEXT: mov v4.s[0], w9
-; CHECK-GI-NEXT: mov v5.s[0], w4
-; CHECK-GI-NEXT: mov v0.s[1], w1
+; CHECK-GI-NEXT: fmov s3, w4
+; CHECK-GI-NEXT: fmov s4, w4
+; CHECK-GI-NEXT: fmov s5, w9
; CHECK-GI-NEXT: mov v1.s[1], w1
+; CHECK-GI-NEXT: mov v0.s[1], w1
; CHECK-GI-NEXT: mov v2.s[1], w8
; CHECK-GI-NEXT: mov v3.s[1], w5
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: mov v5.s[1], w5
-; CHECK-GI-NEXT: mov v0.s[2], w2
+; CHECK-GI-NEXT: mov v4.s[1], w5
+; CHECK-GI-NEXT: mov v5.s[1], w9
; CHECK-GI-NEXT: mov v1.s[2], w2
+; CHECK-GI-NEXT: mov v0.s[2], w2
; CHECK-GI-NEXT: mov v2.s[2], w8
; CHECK-GI-NEXT: mov v3.s[2], w6
-; CHECK-GI-NEXT: mov v4.s[2], w9
-; CHECK-GI-NEXT: mov v5.s[2], w6
-; CHECK-GI-NEXT: mov v0.s[3], w3
+; CHECK-GI-NEXT: mov v4.s[2], w6
+; CHECK-GI-NEXT: mov v5.s[2], w9
; CHECK-GI-NEXT: mov v1.s[3], w3
+; CHECK-GI-NEXT: mov v0.s[3], w3
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: ushl v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT: ushr v1.4s, v1.4s, #29
-; CHECK-GI-NEXT: ushl v2.4s, v5.4s, v2.4s
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: orr v1.16b, v3.16b, v2.16b
+; CHECK-GI-NEXT: ushl v4.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: shl v1.4s, v1.4s, #3
+; CHECK-GI-NEXT: ushr v0.4s, v0.4s, #29
+; CHECK-GI-NEXT: ushl v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT: orr v1.16b, v4.16b, v2.16b
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v0.s[2]
; CHECK-GI-NEXT: mov s4, v0.s[3]
@@ -3664,35 +3664,35 @@ define <7 x i32> @rotr_v7i32_c(<7 x i32> %a) {
;
; CHECK-GI-LABEL: rotr_v7i32_c:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v1.s[0], w0
+; CHECK-GI-NEXT: fmov s0, w0
+; CHECK-GI-NEXT: fmov s1, w0
; CHECK-GI-NEXT: mov w8, #3 // =0x3
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov w9, #29 // =0x1d
-; CHECK-GI-NEXT: mov v3.s[0], w4
-; CHECK-GI-NEXT: mov v4.s[0], w4
-; CHECK-GI-NEXT: mov v5.s[0], w9
-; CHECK-GI-NEXT: mov v0.s[1], w1
+; CHECK-GI-NEXT: fmov s3, w4
+; CHECK-GI-NEXT: fmov s4, w4
+; CHECK-GI-NEXT: fmov s5, w9
; CHECK-GI-NEXT: mov v1.s[1], w1
+; CHECK-GI-NEXT: mov v0.s[1], w1
; CHECK-GI-NEXT: mov v2.s[1], w8
; CHECK-GI-NEXT: mov v3.s[1], w5
; CHECK-GI-NEXT: mov v4.s[1], w5
; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: mov v0.s[2], w2
; CHECK-GI-NEXT: mov v1.s[2], w2
+; CHECK-GI-NEXT: mov v0.s[2], w2
; CHECK-GI-NEXT: mov v2.s[2], w8
; CHECK-GI-NEXT: mov v3.s[2], w6
; CHECK-GI-NEXT: mov v4.s[2], w6
; CHECK-GI-NEXT: mov v5.s[2], w9
-; CHECK-GI-NEXT: mov v0.s[3], w3
; CHECK-GI-NEXT: mov v1.s[3], w3
+; CHECK-GI-NEXT: mov v0.s[3], w3
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: ushl v4.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT: ushr v0.4s, v0.4s, #3
-; CHECK-GI-NEXT: shl v1.4s, v1.4s, #29
-; CHECK-GI-NEXT: ushl v2.4s, v3.4s, v2.4s
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT: orr v1.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT: ushl v3.4s, v3.4s, v5.4s
+; CHECK-GI-NEXT: ushr v1.4s, v1.4s, #3
+; CHECK-GI-NEXT: shl v0.4s, v0.4s, #29
+; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT: orr v1.16b, v2.16b, v3.16b
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v0.s[2]
; CHECK-GI-NEXT: mov s4, v0.s[3]
@@ -4200,39 +4200,39 @@ define <7 x i32> @fshl_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
;
; CHECK-GI-LABEL: fshl_v7i32_c:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
+; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: mov w8, #29 // =0x1d
-; CHECK-GI-NEXT: mov v2.s[0], w7
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s5, w7
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: ldr s1, [sp]
; CHECK-GI-NEXT: mov w9, #3 // =0x3
-; CHECK-GI-NEXT: mov v4.s[0], w4
-; CHECK-GI-NEXT: mov v5.s[0], w9
-; CHECK-GI-NEXT: ldr s3, [sp]
-; CHECK-GI-NEXT: ldr s6, [sp, #24]
-; CHECK-GI-NEXT: ldr s7, [sp, #32]
+; CHECK-GI-NEXT: fmov s7, w9
+; CHECK-GI-NEXT: ldr s4, [sp, #24]
+; CHECK-GI-NEXT: ldr s6, [sp, #32]
; CHECK-GI-NEXT: mov v0.s[1], w1
-; CHECK-GI-NEXT: mov v2.s[1], v3.s[0]
-; CHECK-GI-NEXT: ldr s3, [sp, #8]
-; CHECK-GI-NEXT: mov v1.s[1], w8
-; CHECK-GI-NEXT: mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT: mov v4.s[1], w5
-; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: ldr s7, [sp, #40]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[0]
+; CHECK-GI-NEXT: fmov s1, w4
+; CHECK-GI-NEXT: mov v3.s[1], w8
+; CHECK-GI-NEXT: mov v4.s[1], v6.s[0]
+; CHECK-GI-NEXT: ldr s2, [sp, #8]
+; CHECK-GI-NEXT: mov v7.s[1], w9
+; CHECK-GI-NEXT: ldr s6, [sp, #40]
+; CHECK-GI-NEXT: mov v1.s[1], w5
; CHECK-GI-NEXT: mov v0.s[2], w2
-; CHECK-GI-NEXT: mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT: ldr s3, [sp, #16]
-; CHECK-GI-NEXT: mov v1.s[2], w8
-; CHECK-GI-NEXT: mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT: mov v4.s[2], w6
-; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: mov v5.s[2], v2.s[0]
+; CHECK-GI-NEXT: ldr s2, [sp, #16]
+; CHECK-GI-NEXT: mov v3.s[2], w8
+; CHECK-GI-NEXT: mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT: mov v7.s[2], w9
+; CHECK-GI-NEXT: mov v1.s[2], w6
; CHECK-GI-NEXT: mov v0.s[3], w3
-; CHECK-GI-NEXT: mov v2.s[3], v3.s[0]
-; CHECK-GI-NEXT: neg v1.4s, v1.4s
-; CHECK-GI-NEXT: ushl v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mov v5.s[3], v2.s[0]
+; CHECK-GI-NEXT: neg v3.4s, v3.4s
+; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v7.4s
; CHECK-GI-NEXT: shl v0.4s, v0.4s, #3
-; CHECK-GI-NEXT: ushl v1.4s, v6.4s, v1.4s
-; CHECK-GI-NEXT: usra v0.4s, v2.4s, #29
-; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT: usra v0.4s, v5.4s, #29
+; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v0.s[2]
; CHECK-GI-NEXT: mov s4, v0.s[3]
@@ -4293,39 +4293,39 @@ define <7 x i32> @fshr_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
;
; CHECK-GI-LABEL: fshr_v7i32_c:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
+; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: mov w8, #3 // =0x3
-; CHECK-GI-NEXT: mov v2.s[0], w7
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s5, w7
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: ldr s1, [sp]
; CHECK-GI-NEXT: mov w9, #29 // =0x1d
-; CHECK-GI-NEXT: mov v4.s[0], w4
-; CHECK-GI-NEXT: mov v5.s[0], w9
-; CHECK-GI-NEXT: ldr s3, [sp]
-; CHECK-GI-NEXT: ldr s6, [sp, #24]
-; CHECK-GI-NEXT: ldr s7, [sp, #32]
+; CHECK-GI-NEXT: fmov s7, w9
+; CHECK-GI-NEXT: ldr s4, [sp, #24]
+; CHECK-GI-NEXT: ldr s6, [sp, #32]
; CHECK-GI-NEXT: mov v0.s[1], w1
-; CHECK-GI-NEXT: mov v2.s[1], v3.s[0]
-; CHECK-GI-NEXT: ldr s3, [sp, #8]
-; CHECK-GI-NEXT: mov v1.s[1], w8
-; CHECK-GI-NEXT: mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT: mov v4.s[1], w5
-; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: ldr s7, [sp, #40]
+; CHECK-GI-NEXT: mov v5.s[1], v1.s[0]
+; CHECK-GI-NEXT: fmov s1, w4
+; CHECK-GI-NEXT: mov v3.s[1], w8
+; CHECK-GI-NEXT: mov v4.s[1], v6.s[0]
+; CHECK-GI-NEXT: ldr s2, [sp, #8]
+; CHECK-GI-NEXT: mov v7.s[1], w9
+; CHECK-GI-NEXT: ldr s6, [sp, #40]
+; CHECK-GI-NEXT: mov v1.s[1], w5
; CHECK-GI-NEXT: mov v0.s[2], w2
-; CHECK-GI-NEXT: mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT: ldr s3, [sp, #16]
-; CHECK-GI-NEXT: mov v1.s[2], w8
-; CHECK-GI-NEXT: mov v6.s[2], v7.s[0]
-; CHECK-GI-NEXT: mov v4.s[2], w6
-; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: mov v5.s[2], v2.s[0]
+; CHECK-GI-NEXT: ldr s2, [sp, #16]
+; CHECK-GI-NEXT: mov v3.s[2], w8
+; CHECK-GI-NEXT: mov v4.s[2], v6.s[0]
+; CHECK-GI-NEXT: mov v7.s[2], w9
+; CHECK-GI-NEXT: mov v1.s[2], w6
; CHECK-GI-NEXT: mov v0.s[3], w3
-; CHECK-GI-NEXT: mov v2.s[3], v3.s[0]
-; CHECK-GI-NEXT: neg v1.4s, v1.4s
-; CHECK-GI-NEXT: ushl v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mov v5.s[3], v2.s[0]
+; CHECK-GI-NEXT: neg v3.4s, v3.4s
+; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v7.4s
; CHECK-GI-NEXT: shl v0.4s, v0.4s, #29
-; CHECK-GI-NEXT: ushl v1.4s, v6.4s, v1.4s
-; CHECK-GI-NEXT: usra v0.4s, v2.4s, #3
-; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b
+; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v3.4s
+; CHECK-GI-NEXT: usra v0.4s, v5.4s, #3
+; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v0.s[2]
; CHECK-GI-NEXT: mov s4, v0.s[3]
diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll
index e284795760c5c..18665bcbeae83 100644
--- a/llvm/test/CodeGen/AArch64/icmp.ll
+++ b/llvm/test/CodeGen/AArch64/icmp.ll
@@ -1228,18 +1228,18 @@ define <3 x i32> @v3i32_i32(<3 x i32> %a, <3 x i32> %b, <3 x i32> %d, <3 x i32>
; CHECK-GI-LABEL: v3i32_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #31 // =0x1f
-; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
; CHECK-GI-NEXT: cmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov v5.s[0], w9
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: mov v5.s[1], w9
; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: mov v1.s[1], w8
; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: neg v1.4s, v4.4s
-; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: eor v1.16b, v0.16b, v5.16b
+; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: mov v1.s[2], w8
+; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll
index aa4f31fb5f53e..4c34c2f280959 100644
--- a/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -964,21 +964,13 @@ entry:
}
define <3 x i32> @insert_v3i32_0(<3 x i32> %a, i32 %b, i32 %c) {
-; CHECK-SD-LABEL: insert_v3i32_0:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-SD-NEXT: mov v1.s[2], v0.s[2]
-; CHECK-SD-NEXT: mov v0.16b, v1.16b
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: insert_v3i32_0:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v1.s[0], w0
-; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT: mov v1.s[2], v0.s[2]
-; CHECK-GI-NEXT: mov v0.16b, v1.16b
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: insert_v3i32_0:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov s1, w0
+; CHECK-NEXT: mov v1.s[1], v0.s[1]
+; CHECK-NEXT: mov v1.s[2], v0.s[2]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
entry:
%d = insertelement <3 x i32> %a, i32 %b, i32 0
ret <3 x i32> %d
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index 9d4d654259a31..e8194b9bd9b27 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -3315,11 +3315,11 @@ define <3 x double> @stofp_v3i8_v3f64(<3 x i8> %a) {
; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8
; CHECK-GI-NEXT: smov x8, v0.h[0]
; CHECK-GI-NEXT: smov x9, v0.h[1]
-; CHECK-GI-NEXT: mov v1.d[0], x8
+; CHECK-GI-NEXT: fmov d1, x8
; CHECK-GI-NEXT: smov x8, v0.h[2]
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: smov x9, v0.h[3]
-; CHECK-GI-NEXT: mov v2.d[0], x8
+; CHECK-GI-NEXT: fmov d2, x8
; CHECK-GI-NEXT: scvtf v0.2d, v1.2d
; CHECK-GI-NEXT: mov v2.d[1], x9
; CHECK-GI-NEXT: mov d1, v0.d[1]
@@ -3360,11 +3360,11 @@ define <3 x double> @utofp_v3i8_v3f64(<3 x i8> %a) {
; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT: umov w8, v0.h[0]
; CHECK-GI-NEXT: umov w9, v0.h[1]
-; CHECK-GI-NEXT: mov v1.d[0], x8
+; CHECK-GI-NEXT: fmov d1, x8
; CHECK-GI-NEXT: umov w8, v0.h[2]
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: umov w9, v0.h[3]
-; CHECK-GI-NEXT: mov v2.d[0], x8
+; CHECK-GI-NEXT: fmov d2, x8
; CHECK-GI-NEXT: ucvtf v0.2d, v1.2d
; CHECK-GI-NEXT: mov v2.d[1], x9
; CHECK-GI-NEXT: mov d1, v0.d[1]
diff --git a/llvm/test/CodeGen/AArch64/mul.ll b/llvm/test/CodeGen/AArch64/mul.ll
index 39ce70f88d6ba..1558043f7f40a 100644
--- a/llvm/test/CodeGen/AArch64/mul.ll
+++ b/llvm/test/CodeGen/AArch64/mul.ll
@@ -404,14 +404,14 @@ define <2 x i64> @v2i64(<2 x i64> %d, <2 x i64> %e) {
;
; CHECK-GI-LABEL: v2i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v1.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v1.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
entry:
%s = mul <2 x i64> %d, %e
@@ -449,16 +449,16 @@ define <3 x i64> @v3i64(<3 x i64> %d, <3 x i64> %e) {
; CHECK-GI-NEXT: // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d3
-; CHECK-GI-NEXT: mov x10, v0.d[1]
-; CHECK-GI-NEXT: mov x11, v3.d[1]
+; CHECK-GI-NEXT: fmov x10, d0
+; CHECK-GI-NEXT: fmov x11, d3
+; CHECK-GI-NEXT: mov x8, v0.d[1]
+; CHECK-GI-NEXT: mov x9, v3.d[1]
+; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x10, x11
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: fmov x8, d2
-; CHECK-GI-NEXT: mov v0.d[1], x9
; CHECK-GI-NEXT: fmov x9, d5
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: fmov x8, d2
; CHECK-GI-NEXT: mul x8, x8, x9
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -503,10 +503,10 @@ define <4 x i64> @v4i64(<4 x i64> %d, <4 x i64> %e) {
; CHECK-GI-NEXT: fmov x9, d1
; CHECK-GI-NEXT: mul x10, x10, x11
; CHECK-GI-NEXT: mul x9, x9, x12
-; CHECK-GI-NEXT: mov v0.d[0], x8
+; CHECK-GI-NEXT: fmov d0, x8
; CHECK-GI-NEXT: mul x11, x13, x14
-; CHECK-GI-NEXT: mov v1.d[0], x9
; CHECK-GI-NEXT: mov v0.d[1], x10
+; CHECK-GI-NEXT: fmov d1, x9
; CHECK-GI-NEXT: mov v1.d[1], x11
; CHECK-GI-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
index fb65a748c865f..7e6f3548bdaf5 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
@@ -1143,7 +1143,7 @@ define <4 x i32> @vselect_constant_cond_zero_v4i32(<4 x i32> %a) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
; CHECK-GI-NEXT: mov w9, #0 // =0x0
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: mov v1.s[1], w9
; CHECK-GI-NEXT: mov v1.s[2], w9
; CHECK-GI-NEXT: mov v1.s[3], w8
@@ -1215,7 +1215,7 @@ define <4 x i32> @vselect_constant_cond_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #1 // =0x1
; CHECK-GI-NEXT: mov w9, #0 // =0x0
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mov v2.s[2], w9
; CHECK-GI-NEXT: mov v2.s[3], w8
diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
index 2c2cb72112879..fb8b721769b2b 100644
--- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -2585,7 +2585,7 @@ define <4 x i32> @fcmnv4xfloat(<4 x float> %A, <4 x float> %B) {
; CHECK-GI-LABEL: fcmnv4xfloat:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: mov w8, #0 // =0x0
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: mov v0.s[1], w8
; CHECK-GI-NEXT: mov v0.d[1], v0.d[0]
; CHECK-GI-NEXT: shl v0.4s, v0.4s, #31
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 2d3fda704908e..a534112b7c559 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -415,17 +415,17 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
; CHECK-GI-NEXT: ldrb w8, [x0, #4]
; CHECK-GI-NEXT: ldrb w9, [x1, #4]
; CHECK-GI-NEXT: ldrb w10, [x1]
+; CHECK-GI-NEXT: ldrb w11, [x0, #1]
+; CHECK-GI-NEXT: ldrb w12, [x1, #1]
; CHECK-GI-NEXT: mul w8, w9, w8
; CHECK-GI-NEXT: ldrb w9, [x0]
-; CHECK-GI-NEXT: mov v0.s[0], w10
-; CHECK-GI-NEXT: mov v1.s[0], w9
-; CHECK-GI-NEXT: ldrb w9, [x1, #1]
-; CHECK-GI-NEXT: mov v2.s[0], w8
-; CHECK-GI-NEXT: ldrb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: fmov s0, w10
+; CHECK-GI-NEXT: fmov s1, w9
; CHECK-GI-NEXT: ldrb w9, [x1, #2]
-; CHECK-GI-NEXT: mov v1.s[1], w8
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: mov v0.s[1], w12
; CHECK-GI-NEXT: ldrb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v1.s[1], w11
; CHECK-GI-NEXT: mov v2.s[1], wzr
; CHECK-GI-NEXT: mov v0.s[2], w9
; CHECK-GI-NEXT: ldrb w9, [x1, #3]
@@ -468,12 +468,12 @@ define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ldrb w8, [x0]
; CHECK-GI-NEXT: ldrb w9, [x0, #4]
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: mov v1.s[0], w9
-; CHECK-GI-NEXT: ldrb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v0.s[1], w8
-; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: ldrb w10, [x0, #1]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: fmov s1, w9
; CHECK-GI-NEXT: ldrb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v0.s[1], w10
+; CHECK-GI-NEXT: mov v1.s[1], wzr
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: mov v1.s[2], wzr
; CHECK-GI-NEXT: ldrb w8, [x0, #3]
@@ -509,17 +509,17 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b,
; CHECK-GI-NEXT: ldrsb w8, [x0, #4]
; CHECK-GI-NEXT: ldrsb w9, [x1, #4]
; CHECK-GI-NEXT: ldrsb w10, [x1]
+; CHECK-GI-NEXT: ldrsb w11, [x0, #1]
+; CHECK-GI-NEXT: ldrsb w12, [x1, #1]
; CHECK-GI-NEXT: mul w8, w9, w8
; CHECK-GI-NEXT: ldrsb w9, [x0]
-; CHECK-GI-NEXT: mov v0.s[0], w10
-; CHECK-GI-NEXT: mov v1.s[0], w9
-; CHECK-GI-NEXT: ldrsb w9, [x1, #1]
-; CHECK-GI-NEXT: mov v2.s[0], w8
-; CHECK-GI-NEXT: ldrsb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: fmov s0, w10
+; CHECK-GI-NEXT: fmov s1, w9
; CHECK-GI-NEXT: ldrsb w9, [x1, #2]
-; CHECK-GI-NEXT: mov v1.s[1], w8
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: mov v0.s[1], w12
; CHECK-GI-NEXT: ldrsb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v1.s[1], w11
; CHECK-GI-NEXT: mov v2.s[1], wzr
; CHECK-GI-NEXT: mov v0.s[2], w9
; CHECK-GI-NEXT: ldrsb w9, [x1, #3]
@@ -569,55 +569,55 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3
-; CHECK-GI-NEXT: smov w8, v0.b[4]
-; CHECK-GI-NEXT: smov w9, v1.b[4]
-; CHECK-GI-NEXT: smov w10, v2.b[4]
-; CHECK-GI-NEXT: smov w11, v3.b[4]
-; CHECK-GI-NEXT: smov w12, v0.b[0]
-; CHECK-GI-NEXT: smov w13, v1.b[0]
-; CHECK-GI-NEXT: smov w14, v2.b[0]
-; CHECK-GI-NEXT: smov w15, v3.b[0]
-; CHECK-GI-NEXT: mul w8, w8, w9
-; CHECK-GI-NEXT: smov w9, v0.b[1]
-; CHECK-GI-NEXT: mul w10, w10, w11
-; CHECK-GI-NEXT: smov w11, v1.b[1]
-; CHECK-GI-NEXT: mov v4.s[0], w12
-; CHECK-GI-NEXT: smov w12, v2.b[1]
-; CHECK-GI-NEXT: mov v5.s[0], w13
+; CHECK-GI-NEXT: smov w9, v1.b[0]
+; CHECK-GI-NEXT: smov w10, v0.b[4]
+; CHECK-GI-NEXT: smov w11, v1.b[4]
+; CHECK-GI-NEXT: smov w12, v2.b[0]
+; CHECK-GI-NEXT: smov w13, v2.b[4]
+; CHECK-GI-NEXT: smov w14, v3.b[4]
+; CHECK-GI-NEXT: smov w8, v0.b[0]
+; CHECK-GI-NEXT: smov w16, v3.b[0]
+; CHECK-GI-NEXT: smov w15, v0.b[1]
+; CHECK-GI-NEXT: fmov s5, w9
+; CHECK-GI-NEXT: mul w9, w10, w11
+; CHECK-GI-NEXT: smov w10, v1.b[1]
+; CHECK-GI-NEXT: fmov s6, w12
+; CHECK-GI-NEXT: mul w12, w13, w14
+; CHECK-GI-NEXT: smov w11, v2.b[1]
; CHECK-GI-NEXT: smov w13, v3.b[1]
-; CHECK-GI-NEXT: mov v6.s[0], w8
-; CHECK-GI-NEXT: mov v7.s[0], w14
-; CHECK-GI-NEXT: mov v16.s[0], w15
-; CHECK-GI-NEXT: mov v17.s[0], w10
+; CHECK-GI-NEXT: fmov s4, w8
+; CHECK-GI-NEXT: fmov s7, w16
+; CHECK-GI-NEXT: fmov s16, w9
; CHECK-GI-NEXT: smov w8, v0.b[2]
-; CHECK-GI-NEXT: smov w10, v1.b[2]
-; CHECK-GI-NEXT: smov w14, v2.b[2]
-; CHECK-GI-NEXT: smov w15, v3.b[2]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: mov v5.s[1], w11
-; CHECK-GI-NEXT: smov w9, v0.b[3]
-; CHECK-GI-NEXT: smov w11, v1.b[3]
-; CHECK-GI-NEXT: mov v6.s[1], wzr
-; CHECK-GI-NEXT: mov v7.s[1], w12
-; CHECK-GI-NEXT: mov v16.s[1], w13
+; CHECK-GI-NEXT: smov w14, v1.b[2]
+; CHECK-GI-NEXT: fmov s17, w12
+; CHECK-GI-NEXT: smov w9, v3.b[2]
+; CHECK-GI-NEXT: mov v5.s[1], w10
+; CHECK-GI-NEXT: mov v4.s[1], w15
+; CHECK-GI-NEXT: smov w15, v2.b[2]
+; CHECK-GI-NEXT: mov v6.s[1], w11
+; CHECK-GI-NEXT: mov v16.s[1], wzr
+; CHECK-GI-NEXT: mov v7.s[1], w13
+; CHECK-GI-NEXT: smov w10, v0.b[3]
; CHECK-GI-NEXT: mov v17.s[1], wzr
+; CHECK-GI-NEXT: smov w11, v1.b[3]
; CHECK-GI-NEXT: smov w12, v2.b[3]
; CHECK-GI-NEXT: smov w13, v3.b[3]
+; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: mov v5.s[2], w10
-; CHECK-GI-NEXT: mov v6.s[2], wzr
-; CHECK-GI-NEXT: mov v7.s[2], w14
-; CHECK-GI-NEXT: mov v16.s[2], w15
+; CHECK-GI-NEXT: mov v6.s[2], w15
+; CHECK-GI-NEXT: mov v16.s[2], wzr
+; CHECK-GI-NEXT: mov v7.s[2], w9
; CHECK-GI-NEXT: mov v17.s[2], wzr
-; CHECK-GI-NEXT: mov v4.s[3], w9
; CHECK-GI-NEXT: mov v5.s[3], w11
-; CHECK-GI-NEXT: mov v6.s[3], wzr
-; CHECK-GI-NEXT: mov v7.s[3], w12
-; CHECK-GI-NEXT: mov v16.s[3], w13
+; CHECK-GI-NEXT: mov v4.s[3], w10
+; CHECK-GI-NEXT: mov v6.s[3], w12
+; CHECK-GI-NEXT: mov v16.s[3], wzr
+; CHECK-GI-NEXT: mov v7.s[3], w13
; CHECK-GI-NEXT: mov v17.s[3], wzr
-; CHECK-GI-NEXT: mla v6.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT: mla v17.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT: addv s0, v6.4s
+; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mla v17.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: addv s0, v16.4s
; CHECK-GI-NEXT: addv s1, v17.4s
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
@@ -661,26 +661,26 @@ define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <
; CHECK-GI-NEXT: smov w8, v0.b[0]
; CHECK-GI-NEXT: smov w9, v0.b[4]
; CHECK-GI-NEXT: smov w10, v2.b[0]
-; CHECK-GI-NEXT: smov w11, v2.b[4]
-; CHECK-GI-NEXT: smov w12, v0.b[1]
-; CHECK-GI-NEXT: mov v1.s[0], w8
-; CHECK-GI-NEXT: smov w8, v2.b[1]
-; CHECK-GI-NEXT: mov v3.s[0], w9
-; CHECK-GI-NEXT: mov v4.s[0], w10
-; CHECK-GI-NEXT: mov v5.s[0], w11
-; CHECK-GI-NEXT: smov w9, v0.b[2]
-; CHECK-GI-NEXT: smov w10, v2.b[2]
-; CHECK-GI-NEXT: smov w11, v2.b[3]
-; CHECK-GI-NEXT: mov v1.s[1], w12
+; CHECK-GI-NEXT: smov w12, v2.b[4]
+; CHECK-GI-NEXT: smov w11, v0.b[1]
+; CHECK-GI-NEXT: smov w13, v2.b[1]
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: fmov s3, w9
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: fmov s5, w12
+; CHECK-GI-NEXT: smov w8, v0.b[2]
+; CHECK-GI-NEXT: smov w9, v2.b[2]
+; CHECK-GI-NEXT: smov w10, v0.b[3]
+; CHECK-GI-NEXT: mov v1.s[1], w11
; CHECK-GI-NEXT: mov v3.s[1], wzr
-; CHECK-GI-NEXT: mov v4.s[1], w8
+; CHECK-GI-NEXT: mov v4.s[1], w13
; CHECK-GI-NEXT: mov v5.s[1], wzr
-; CHECK-GI-NEXT: smov w8, v0.b[3]
-; CHECK-GI-NEXT: mov v1.s[2], w9
+; CHECK-GI-NEXT: smov w11, v2.b[3]
+; CHECK-GI-NEXT: mov v1.s[2], w8
; CHECK-GI-NEXT: mov v3.s[2], wzr
-; CHECK-GI-NEXT: mov v4.s[2], w10
+; CHECK-GI-NEXT: mov v4.s[2], w9
; CHECK-GI-NEXT: mov v5.s[2], wzr
-; CHECK-GI-NEXT: mov v1.s[3], w8
+; CHECK-GI-NEXT: mov v1.s[3], w10
; CHECK-GI-NEXT: mov v3.s[3], wzr
; CHECK-GI-NEXT: mov v4.s[3], w11
; CHECK-GI-NEXT: mov v5.s[3], wzr
@@ -2298,125 +2298,125 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_udot_v25i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldr q1, [x1]
-; CHECK-GI-NEXT: ldrb w11, [x1, #16]!
-; CHECK-GI-NEXT: ldrb w12, [x1, #4]
-; CHECK-GI-NEXT: ldr q0, [x0]
-; CHECK-GI-NEXT: mov v23.s[0], wzr
-; CHECK-GI-NEXT: umov w9, v1.b[4]
-; CHECK-GI-NEXT: umov w10, v1.b[12]
-; CHECK-GI-NEXT: umov w13, v1.b[0]
-; CHECK-GI-NEXT: umov w14, v1.b[5]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov v3.s[0], w11
-; CHECK-GI-NEXT: umov w11, v0.b[0]
-; CHECK-GI-NEXT: umov w12, v1.b[1]
-; CHECK-GI-NEXT: umov w15, v1.b[8]
-; CHECK-GI-NEXT: ldrb w8, [x0, #16]!
-; CHECK-GI-NEXT: mov v23.s[1], wzr
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: mov v4.s[0], w10
+; CHECK-GI-NEXT: ldr q2, [x1]
+; CHECK-GI-NEXT: ldrb w9, [x1, #16]!
+; CHECK-GI-NEXT: ldrb w11, [x1, #4]
+; CHECK-GI-NEXT: ldrb w12, [x1, #5]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: umov w13, v2.b[4]
+; CHECK-GI-NEXT: umov w14, v2.b[5]
+; CHECK-GI-NEXT: umov w10, v2.b[0]
+; CHECK-GI-NEXT: fmov s3, w9
+; CHECK-GI-NEXT: umov w9, v2.b[8]
+; CHECK-GI-NEXT: fmov s5, w11
+; CHECK-GI-NEXT: umov w11, v2.b[12]
+; CHECK-GI-NEXT: ldr q1, [x0]
+; CHECK-GI-NEXT: ldrb w8, [x1, #1]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: fmov s7, w13
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: umov w10, v2.b[13]
+; CHECK-GI-NEXT: mov v5.s[1], w12
+; CHECK-GI-NEXT: umov w13, v2.b[9]
+; CHECK-GI-NEXT: fmov s6, w9
+; CHECK-GI-NEXT: fmov s16, w11
+; CHECK-GI-NEXT: umov w9, v1.b[0]
+; CHECK-GI-NEXT: mov v3.s[1], w8
+; CHECK-GI-NEXT: mov v7.s[1], w14
+; CHECK-GI-NEXT: umov w14, v2.b[6]
+; CHECK-GI-NEXT: ldrb w12, [x1, #6]
+; CHECK-GI-NEXT: umov w8, v2.b[1]
+; CHECK-GI-NEXT: umov w11, v2.b[2]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: umov w10, v2.b[14]
+; CHECK-GI-NEXT: mov v5.s[2], w12
+; CHECK-GI-NEXT: umov w12, v1.b[5]
+; CHECK-GI-NEXT: mov v6.s[1], w13
+; CHECK-GI-NEXT: fmov s17, w9
+; CHECK-GI-NEXT: mov v7.s[2], w14
+; CHECK-GI-NEXT: umov w14, v1.b[4]
+; CHECK-GI-NEXT: umov w9, v2.b[10]
+; CHECK-GI-NEXT: mov v4.s[1], w8
+; CHECK-GI-NEXT: umov w8, v1.b[1]
+; CHECK-GI-NEXT: umov w13, v2.b[7]
+; CHECK-GI-NEXT: mov v16.s[2], w10
+; CHECK-GI-NEXT: umov w10, v2.b[15]
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: fmov s18, w14
+; CHECK-GI-NEXT: mov v6.s[2], w9
+; CHECK-GI-NEXT: umov w9, v1.b[12]
+; CHECK-GI-NEXT: mov v4.s[2], w11
+; CHECK-GI-NEXT: ldrb w11, [x1, #7]
+; CHECK-GI-NEXT: mov v17.s[1], w8
+; CHECK-GI-NEXT: ldrb w8, [x1, #2]
+; CHECK-GI-NEXT: mov v16.s[3], w10
; CHECK-GI-NEXT: umov w10, v1.b[13]
-; CHECK-GI-NEXT: ldrb w9, [x1, #5]
-; CHECK-GI-NEXT: mov v6.s[0], w13
-; CHECK-GI-NEXT: umov w13, v1.b[6]
-; CHECK-GI-NEXT: mov v16.s[0], w11
-; CHECK-GI-NEXT: umov w11, v1.b[2]
-; CHECK-GI-NEXT: mov v7.s[0], w15
-; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: ldrb w9, [x1, #6]
-; CHECK-GI-NEXT: umov w15, v1.b[9]
-; CHECK-GI-NEXT: mov v2.s[1], w14
-; CHECK-GI-NEXT: ldrb w14, [x1, #1]
-; CHECK-GI-NEXT: mov v4.s[1], w10
-; CHECK-GI-NEXT: umov w10, v1.b[14]
-; CHECK-GI-NEXT: mov v6.s[1], w12
-; CHECK-GI-NEXT: umov w12, v0.b[1]
-; CHECK-GI-NEXT: mov v3.s[1], w14
-; CHECK-GI-NEXT: umov w14, v0.b[12]
-; CHECK-GI-NEXT: mov v21.s[0], w8
-; CHECK-GI-NEXT: ldrb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v5.s[2], w9
-; CHECK-GI-NEXT: umov w9, v0.b[4]
-; CHECK-GI-NEXT: mov v2.s[2], w13
-; CHECK-GI-NEXT: umov w13, v1.b[7]
-; CHECK-GI-NEXT: mov v7.s[1], w15
-; CHECK-GI-NEXT: mov v4.s[2], w10
-; CHECK-GI-NEXT: umov w10, v1.b[15]
-; CHECK-GI-NEXT: mov v16.s[1], w12
-; CHECK-GI-NEXT: ldrb w12, [x1, #2]
-; CHECK-GI-NEXT: mov v6.s[2], w11
-; CHECK-GI-NEXT: umov w11, v0.b[2]
-; CHECK-GI-NEXT: mov v17.s[0], w9
-; CHECK-GI-NEXT: umov w9, v0.b[8]
-; CHECK-GI-NEXT: mov v18.s[0], w14
-; CHECK-GI-NEXT: mov v2.s[3], w13
-; CHECK-GI-NEXT: ldrb w13, [x1, #7]
-; CHECK-GI-NEXT: mov v3.s[2], w12
+; CHECK-GI-NEXT: mov v18.s[1], w12
+; CHECK-GI-NEXT: umov w12, v1.b[6]
+; CHECK-GI-NEXT: mov v5.s[3], w11
+; CHECK-GI-NEXT: ldrb w11, [x0, #16]!
+; CHECK-GI-NEXT: mov v7.s[3], w13
+; CHECK-GI-NEXT: umov w13, v1.b[2]
+; CHECK-GI-NEXT: fmov s20, w9
+; CHECK-GI-NEXT: ldrb w9, [x0, #5]
+; CHECK-GI-NEXT: mov v3.s[2], w8
+; CHECK-GI-NEXT: umov w8, v1.b[8]
+; CHECK-GI-NEXT: fmov s22, w11
+; CHECK-GI-NEXT: mov v18.s[2], w12
; CHECK-GI-NEXT: ldrb w12, [x0, #4]
-; CHECK-GI-NEXT: mov v4.s[3], w10
-; CHECK-GI-NEXT: umov w10, v0.b[5]
-; CHECK-GI-NEXT: mov v5.s[3], w13
-; CHECK-GI-NEXT: ldrb w13, [x0, #8]
-; CHECK-GI-NEXT: mov v16.s[2], w11
-; CHECK-GI-NEXT: umov w11, v0.b[13]
-; CHECK-GI-NEXT: mov v20.s[0], w12
-; CHECK-GI-NEXT: ldrb w12, [x1, #8]
-; CHECK-GI-NEXT: mov v19.s[0], w9
-; CHECK-GI-NEXT: umov w9, v0.b[6]
-; CHECK-GI-NEXT: umov w15, v1.b[10]
-; CHECK-GI-NEXT: mul w12, w12, w13
-; CHECK-GI-NEXT: mov v17.s[1], w10
-; CHECK-GI-NEXT: ldrb w10, [x0, #5]
-; CHECK-GI-NEXT: umov w13, v0.b[9]
-; CHECK-GI-NEXT: mov v21.s[1], w8
-; CHECK-GI-NEXT: umov w8, v1.b[11]
-; CHECK-GI-NEXT: mov v18.s[1], w11
-; CHECK-GI-NEXT: umov w11, v0.b[14]
+; CHECK-GI-NEXT: umov w11, v2.b[3]
; CHECK-GI-NEXT: mov v20.s[1], w10
-; CHECK-GI-NEXT: ldrb w10, [x0, #6]
-; CHECK-GI-NEXT: mov v22.s[0], w12
-; CHECK-GI-NEXT: umov w12, v0.b[7]
-; CHECK-GI-NEXT: mov v17.s[2], w9
-; CHECK-GI-NEXT: umov w9, v0.b[10]
-; CHECK-GI-NEXT: mov v7.s[2], w15
+; CHECK-GI-NEXT: ldrb w10, [x0, #8]
+; CHECK-GI-NEXT: fmov s21, w12
+; CHECK-GI-NEXT: ldrb w12, [x1, #8]
+; CHECK-GI-NEXT: mov v17.s[2], w13
+; CHECK-GI-NEXT: umov w13, v1.b[9]
+; CHECK-GI-NEXT: fmov s19, w8
+; CHECK-GI-NEXT: umov w8, v1.b[14]
+; CHECK-GI-NEXT: mul w10, w12, w10
+; CHECK-GI-NEXT: umov w12, v1.b[7]
+; CHECK-GI-NEXT: mov v4.s[3], w11
+; CHECK-GI-NEXT: mov v21.s[1], w9
+; CHECK-GI-NEXT: ldrb w9, [x0, #6]
; CHECK-GI-NEXT: mov v19.s[1], w13
-; CHECK-GI-NEXT: umov w13, v1.b[3]
-; CHECK-GI-NEXT: mov v23.s[2], wzr
-; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: umov w11, v0.b[15]
-; CHECK-GI-NEXT: mov v20.s[2], w10
-; CHECK-GI-NEXT: ldrb w10, [x0, #2]
-; CHECK-GI-NEXT: mov v22.s[1], wzr
-; CHECK-GI-NEXT: mov v17.s[3], w12
+; CHECK-GI-NEXT: ldrb w13, [x0, #1]
+; CHECK-GI-NEXT: mov v20.s[2], w8
+; CHECK-GI-NEXT: umov w8, v1.b[10]
+; CHECK-GI-NEXT: mov v18.s[3], w12
; CHECK-GI-NEXT: ldrb w12, [x0, #7]
-; CHECK-GI-NEXT: mov v7.s[3], w8
-; CHECK-GI-NEXT: ldrb w8, [x0, #3]
-; CHECK-GI-NEXT: mov v19.s[2], w9
-; CHECK-GI-NEXT: umov w9, v0.b[3]
-; CHECK-GI-NEXT: mov v18.s[3], w11
-; CHECK-GI-NEXT: umov w11, v0.b[11]
-; CHECK-GI-NEXT: mov v21.s[2], w10
+; CHECK-GI-NEXT: mov v21.s[2], w9
+; CHECK-GI-NEXT: umov w9, v2.b[11]
+; CHECK-GI-NEXT: fmov s2, w10
+; CHECK-GI-NEXT: ldrb w10, [x0, #2]
+; CHECK-GI-NEXT: mov v22.s[1], w13
+; CHECK-GI-NEXT: umov w13, v1.b[15]
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v19.s[2], w8
+; CHECK-GI-NEXT: umov w8, v1.b[3]
+; CHECK-GI-NEXT: mov v21.s[3], w12
+; CHECK-GI-NEXT: mov v6.s[3], w9
+; CHECK-GI-NEXT: ldrb w9, [x0, #3]
+; CHECK-GI-NEXT: mov v20.s[3], w13
+; CHECK-GI-NEXT: umov w13, v1.b[11]
+; CHECK-GI-NEXT: mov v22.s[2], w10
; CHECK-GI-NEXT: ldrb w10, [x1, #3]
-; CHECK-GI-NEXT: mov v20.s[3], w12
-; CHECK-GI-NEXT: mov v22.s[2], wzr
-; CHECK-GI-NEXT: mov v6.s[3], w13
-; CHECK-GI-NEXT: mul v0.4s, v2.4s, v17.4s
-; CHECK-GI-NEXT: mov v23.s[3], wzr
+; CHECK-GI-NEXT: mul v1.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v17.s[3], w8
; CHECK-GI-NEXT: mov v3.s[3], w10
-; CHECK-GI-NEXT: mov v16.s[3], w9
-; CHECK-GI-NEXT: mov v19.s[3], w11
-; CHECK-GI-NEXT: mul v1.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT: mov v21.s[3], w8
-; CHECK-GI-NEXT: mul v2.4s, v5.4s, v20.4s
-; CHECK-GI-NEXT: mov v22.s[3], wzr
-; CHECK-GI-NEXT: mla v0.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT: mla v1.4s, v7.4s, v19.4s
-; CHECK-GI-NEXT: mla v2.4s, v3.4s, v21.4s
-; CHECK-GI-NEXT: add v3.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mul v5.4s, v5.4s, v21.4s
+; CHECK-GI-NEXT: mov v19.s[3], w13
+; CHECK-GI-NEXT: mul v7.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT: mov v22.s[3], w9
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mla v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT: mla v7.4s, v6.4s, v19.4s
+; CHECK-GI-NEXT: mla v5.4s, v3.4s, v22.4s
+; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v7.4s
+; CHECK-GI-NEXT: add v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT: addv s0, v0.4s
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: add w0, w8, w2
@@ -2456,66 +2456,66 @@ define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
; CHECK-GI-LABEL: test_udot_v25i8_nomla:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ldr q1, [x0]
-; CHECK-GI-NEXT: ldrb w11, [x0, #16]!
-; CHECK-GI-NEXT: ldrb w14, [x0, #4]
-; CHECK-GI-NEXT: ldrb w17, [x0, #8]
-; CHECK-GI-NEXT: mov v0.s[0], wzr
-; CHECK-GI-NEXT: umov w12, v1.b[0]
-; CHECK-GI-NEXT: umov w13, v1.b[4]
-; CHECK-GI-NEXT: umov w15, v1.b[8]
-; CHECK-GI-NEXT: umov w16, v1.b[12]
-; CHECK-GI-NEXT: umov w18, v1.b[1]
-; CHECK-GI-NEXT: umov w1, v1.b[5]
-; CHECK-GI-NEXT: umov w2, v1.b[9]
-; CHECK-GI-NEXT: umov w3, v1.b[13]
-; CHECK-GI-NEXT: mov v4.s[0], w11
-; CHECK-GI-NEXT: mov v7.s[0], w14
-; CHECK-GI-NEXT: mov v16.s[0], w17
-; CHECK-GI-NEXT: ldrb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v2.s[0], w12
-; CHECK-GI-NEXT: mov v3.s[0], w13
-; CHECK-GI-NEXT: ldrb w10, [x0, #5]
-; CHECK-GI-NEXT: mov v5.s[0], w15
-; CHECK-GI-NEXT: mov v6.s[0], w16
+; CHECK-GI-NEXT: ldrb w17, [x0, #16]!
+; CHECK-GI-NEXT: ldrb w16, [x0, #4]
+; CHECK-GI-NEXT: ldrb w14, [x0, #8]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: umov w15, v1.b[0]
+; CHECK-GI-NEXT: umov w2, v1.b[4]
+; CHECK-GI-NEXT: umov w4, v1.b[8]
+; CHECK-GI-NEXT: umov w5, v1.b[12]
+; CHECK-GI-NEXT: umov w1, v1.b[1]
+; CHECK-GI-NEXT: umov w3, v1.b[5]
+; CHECK-GI-NEXT: umov w6, v1.b[9]
+; CHECK-GI-NEXT: umov w7, v1.b[13]
+; CHECK-GI-NEXT: fmov s6, w17
+; CHECK-GI-NEXT: fmov s7, w16
+; CHECK-GI-NEXT: fmov s16, w14
+; CHECK-GI-NEXT: ldrb w18, [x0, #1]
+; CHECK-GI-NEXT: fmov s2, w15
+; CHECK-GI-NEXT: fmov s3, w2
+; CHECK-GI-NEXT: ldrb w11, [x0, #5]
+; CHECK-GI-NEXT: fmov s4, w4
+; CHECK-GI-NEXT: fmov s5, w5
; CHECK-GI-NEXT: ldrb w16, [x0, #2]
; CHECK-GI-NEXT: umov w9, v1.b[2]
; CHECK-GI-NEXT: umov w12, v1.b[6]
; CHECK-GI-NEXT: ldrb w17, [x0, #6]
-; CHECK-GI-NEXT: umov w14, v1.b[10]
+; CHECK-GI-NEXT: umov w13, v1.b[10]
; CHECK-GI-NEXT: umov w15, v1.b[14]
-; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: mov v2.s[1], w18
-; CHECK-GI-NEXT: mov v3.s[1], w1
-; CHECK-GI-NEXT: mov v7.s[1], w10
-; CHECK-GI-NEXT: mov v5.s[1], w2
-; CHECK-GI-NEXT: mov v6.s[1], w3
+; CHECK-GI-NEXT: mov v2.s[1], w1
+; CHECK-GI-NEXT: mov v3.s[1], w3
+; CHECK-GI-NEXT: mov v4.s[1], w6
+; CHECK-GI-NEXT: mov v5.s[1], w7
+; CHECK-GI-NEXT: mov v6.s[1], w18
+; CHECK-GI-NEXT: mov v7.s[1], w11
; CHECK-GI-NEXT: mov v16.s[1], wzr
; CHECK-GI-NEXT: mov v0.s[1], wzr
-; CHECK-GI-NEXT: umov w11, v1.b[3]
-; CHECK-GI-NEXT: umov w13, v1.b[7]
-; CHECK-GI-NEXT: umov w8, v1.b[11]
-; CHECK-GI-NEXT: umov w10, v1.b[15]
-; CHECK-GI-NEXT: mov v4.s[2], w16
+; CHECK-GI-NEXT: umov w8, v1.b[3]
+; CHECK-GI-NEXT: umov w10, v1.b[7]
+; CHECK-GI-NEXT: umov w11, v1.b[11]
+; CHECK-GI-NEXT: umov w14, v1.b[15]
; CHECK-GI-NEXT: mov v2.s[2], w9
; CHECK-GI-NEXT: ldrb w9, [x0, #3]
; CHECK-GI-NEXT: mov v3.s[2], w12
; CHECK-GI-NEXT: ldrb w12, [x0, #7]
-; CHECK-GI-NEXT: mov v5.s[2], w14
-; CHECK-GI-NEXT: mov v6.s[2], w15
+; CHECK-GI-NEXT: mov v4.s[2], w13
+; CHECK-GI-NEXT: mov v5.s[2], w15
+; CHECK-GI-NEXT: mov v6.s[2], w16
; CHECK-GI-NEXT: mov v7.s[2], w17
; CHECK-GI-NEXT: mov v16.s[2], wzr
; CHECK-GI-NEXT: mov v0.s[2], wzr
-; CHECK-GI-NEXT: mov v4.s[3], w9
-; CHECK-GI-NEXT: mov v2.s[3], w11
-; CHECK-GI-NEXT: mov v3.s[3], w13
-; CHECK-GI-NEXT: mov v5.s[3], w8
-; CHECK-GI-NEXT: mov v6.s[3], w10
+; CHECK-GI-NEXT: mov v2.s[3], w8
+; CHECK-GI-NEXT: mov v3.s[3], w10
+; CHECK-GI-NEXT: mov v4.s[3], w11
+; CHECK-GI-NEXT: mov v5.s[3], w14
+; CHECK-GI-NEXT: mov v6.s[3], w9
; CHECK-GI-NEXT: mov v7.s[3], w12
; CHECK-GI-NEXT: mov v16.s[3], wzr
; CHECK-GI-NEXT: mov v0.s[3], wzr
; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v5.4s, v6.4s
-; CHECK-GI-NEXT: add v3.4s, v4.4s, v7.4s
+; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s
; CHECK-GI-NEXT: add v0.4s, v16.4s, v0.4s
; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
@@ -2554,125 +2554,125 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_sdot_v25i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldr q1, [x1]
-; CHECK-GI-NEXT: ldrsb w11, [x1, #16]!
-; CHECK-GI-NEXT: ldrsb w12, [x1, #4]
-; CHECK-GI-NEXT: ldr q0, [x0]
-; CHECK-GI-NEXT: mov v23.s[0], wzr
-; CHECK-GI-NEXT: smov w9, v1.b[4]
-; CHECK-GI-NEXT: smov w10, v1.b[12]
-; CHECK-GI-NEXT: smov w13, v1.b[0]
-; CHECK-GI-NEXT: smov w14, v1.b[5]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov v3.s[0], w11
-; CHECK-GI-NEXT: smov w11, v0.b[0]
-; CHECK-GI-NEXT: smov w12, v1.b[1]
-; CHECK-GI-NEXT: smov w15, v1.b[8]
-; CHECK-GI-NEXT: ldrsb w8, [x0, #16]!
-; CHECK-GI-NEXT: mov v23.s[1], wzr
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: mov v4.s[0], w10
+; CHECK-GI-NEXT: ldr q2, [x1]
+; CHECK-GI-NEXT: ldrsb w9, [x1, #16]!
+; CHECK-GI-NEXT: ldrsb w11, [x1, #4]
+; CHECK-GI-NEXT: ldrsb w12, [x1, #5]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: smov w13, v2.b[4]
+; CHECK-GI-NEXT: smov w14, v2.b[5]
+; CHECK-GI-NEXT: smov w10, v2.b[0]
+; CHECK-GI-NEXT: fmov s3, w9
+; CHECK-GI-NEXT: smov w9, v2.b[8]
+; CHECK-GI-NEXT: fmov s5, w11
+; CHECK-GI-NEXT: smov w11, v2.b[12]
+; CHECK-GI-NEXT: ldr q1, [x0]
+; CHECK-GI-NEXT: ldrsb w8, [x1, #1]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: fmov s7, w13
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: smov w10, v2.b[13]
+; CHECK-GI-NEXT: mov v5.s[1], w12
+; CHECK-GI-NEXT: smov w13, v2.b[9]
+; CHECK-GI-NEXT: fmov s6, w9
+; CHECK-GI-NEXT: fmov s16, w11
+; CHECK-GI-NEXT: smov w9, v1.b[0]
+; CHECK-GI-NEXT: mov v3.s[1], w8
+; CHECK-GI-NEXT: mov v7.s[1], w14
+; CHECK-GI-NEXT: smov w14, v2.b[6]
+; CHECK-GI-NEXT: ldrsb w12, [x1, #6]
+; CHECK-GI-NEXT: smov w8, v2.b[1]
+; CHECK-GI-NEXT: smov w11, v2.b[2]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: smov w10, v2.b[14]
+; CHECK-GI-NEXT: mov v5.s[2], w12
+; CHECK-GI-NEXT: smov w12, v1.b[5]
+; CHECK-GI-NEXT: mov v6.s[1], w13
+; CHECK-GI-NEXT: fmov s17, w9
+; CHECK-GI-NEXT: mov v7.s[2], w14
+; CHECK-GI-NEXT: smov w14, v1.b[4]
+; CHECK-GI-NEXT: smov w9, v2.b[10]
+; CHECK-GI-NEXT: mov v4.s[1], w8
+; CHECK-GI-NEXT: smov w8, v1.b[1]
+; CHECK-GI-NEXT: smov w13, v2.b[7]
+; CHECK-GI-NEXT: mov v16.s[2], w10
+; CHECK-GI-NEXT: smov w10, v2.b[15]
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: fmov s18, w14
+; CHECK-GI-NEXT: mov v6.s[2], w9
+; CHECK-GI-NEXT: smov w9, v1.b[12]
+; CHECK-GI-NEXT: mov v4.s[2], w11
+; CHECK-GI-NEXT: ldrsb w11, [x1, #7]
+; CHECK-GI-NEXT: mov v17.s[1], w8
+; CHECK-GI-NEXT: ldrsb w8, [x1, #2]
+; CHECK-GI-NEXT: mov v16.s[3], w10
; CHECK-GI-NEXT: smov w10, v1.b[13]
-; CHECK-GI-NEXT: ldrsb w9, [x1, #5]
-; CHECK-GI-NEXT: mov v6.s[0], w13
-; CHECK-GI-NEXT: smov w13, v1.b[6]
-; CHECK-GI-NEXT: mov v16.s[0], w11
-; CHECK-GI-NEXT: smov w11, v1.b[2]
-; CHECK-GI-NEXT: mov v7.s[0], w15
-; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: ldrsb w9, [x1, #6]
-; CHECK-GI-NEXT: smov w15, v1.b[9]
-; CHECK-GI-NEXT: mov v2.s[1], w14
-; CHECK-GI-NEXT: ldrsb w14, [x1, #1]
-; CHECK-GI-NEXT: mov v4.s[1], w10
-; CHECK-GI-NEXT: smov w10, v1.b[14]
-; CHECK-GI-NEXT: mov v6.s[1], w12
-; CHECK-GI-NEXT: smov w12, v0.b[1]
-; CHECK-GI-NEXT: mov v3.s[1], w14
-; CHECK-GI-NEXT: smov w14, v0.b[12]
-; CHECK-GI-NEXT: mov v21.s[0], w8
-; CHECK-GI-NEXT: ldrsb w8, [x0, #1]
-; CHECK-GI-NEXT: mov v5.s[2], w9
-; CHECK-GI-NEXT: smov w9, v0.b[4]
-; CHECK-GI-NEXT: mov v2.s[2], w13
-; CHECK-GI-NEXT: smov w13, v1.b[7]
-; CHECK-GI-NEXT: mov v7.s[1], w15
-; CHECK-GI-NEXT: mov v4.s[2], w10
-; CHECK-GI-NEXT: smov w10, v1.b[15]
-; CHECK-GI-NEXT: mov v16.s[1], w12
-; CHECK-GI-NEXT: ldrsb w12, [x1, #2]
-; CHECK-GI-NEXT: mov v6.s[2], w11
-; CHECK-GI-NEXT: smov w11, v0.b[2]
-; CHECK-GI-NEXT: mov v17.s[0], w9
-; CHECK-GI-NEXT: smov w9, v0.b[8]
-; CHECK-GI-NEXT: mov v18.s[0], w14
-; CHECK-GI-NEXT: mov v2.s[3], w13
-; CHECK-GI-NEXT: ldrsb w13, [x1, #7]
-; CHECK-GI-NEXT: mov v3.s[2], w12
+; CHECK-GI-NEXT: mov v18.s[1], w12
+; CHECK-GI-NEXT: smov w12, v1.b[6]
+; CHECK-GI-NEXT: mov v5.s[3], w11
+; CHECK-GI-NEXT: ldrsb w11, [x0, #16]!
+; CHECK-GI-NEXT: mov v7.s[3], w13
+; CHECK-GI-NEXT: smov w13, v1.b[2]
+; CHECK-GI-NEXT: fmov s20, w9
+; CHECK-GI-NEXT: ldrsb w9, [x0, #5]
+; CHECK-GI-NEXT: mov v3.s[2], w8
+; CHECK-GI-NEXT: smov w8, v1.b[8]
+; CHECK-GI-NEXT: fmov s22, w11
+; CHECK-GI-NEXT: mov v18.s[2], w12
; CHECK-GI-NEXT: ldrsb w12, [x0, #4]
-; CHECK-GI-NEXT: mov v4.s[3], w10
-; CHECK-GI-NEXT: smov w10, v0.b[5]
-; CHECK-GI-NEXT: mov v5.s[3], w13
-; CHECK-GI-NEXT: ldrsb w13, [x0, #8]
-; CHECK-GI-NEXT: mov v16.s[2], w11
-; CHECK-GI-NEXT: smov w11, v0.b[13]
-; CHECK-GI-NEXT: mov v20.s[0], w12
-; CHECK-GI-NEXT: ldrsb w12, [x1, #8]
-; CHECK-GI-NEXT: mov v19.s[0], w9
-; CHECK-GI-NEXT: smov w9, v0.b[6]
-; CHECK-GI-NEXT: smov w15, v1.b[10]
-; CHECK-GI-NEXT: mul w12, w12, w13
-; CHECK-GI-NEXT: mov v17.s[1], w10
-; CHECK-GI-NEXT: ldrsb w10, [x0, #5]
-; CHECK-GI-NEXT: smov w13, v0.b[9]
-; CHECK-GI-NEXT: mov v21.s[1], w8
-; CHECK-GI-NEXT: smov w8, v1.b[11]
-; CHECK-GI-NEXT: mov v18.s[1], w11
-; CHECK-GI-NEXT: smov w11, v0.b[14]
+; CHECK-GI-NEXT: smov w11, v2.b[3]
; CHECK-GI-NEXT: mov v20.s[1], w10
-; CHECK-GI-NEXT: ldrsb w10, [x0, #6]
-; CHECK-GI-NEXT: mov v22.s[0], w12
-; CHECK-GI-NEXT: smov w12, v0.b[7]
-; CHECK-GI-NEXT: mov v17.s[2], w9
-; CHECK-GI-NEXT: smov w9, v0.b[10]
-; CHECK-GI-NEXT: mov v7.s[2], w15
+; CHECK-GI-NEXT: ldrsb w10, [x0, #8]
+; CHECK-GI-NEXT: fmov s21, w12
+; CHECK-GI-NEXT: ldrsb w12, [x1, #8]
+; CHECK-GI-NEXT: mov v17.s[2], w13
+; CHECK-GI-NEXT: smov w13, v1.b[9]
+; CHECK-GI-NEXT: fmov s19, w8
+; CHECK-GI-NEXT: smov w8, v1.b[14]
+; CHECK-GI-NEXT: mul w10, w12, w10
+; CHECK-GI-NEXT: smov w12, v1.b[7]
+; CHECK-GI-NEXT: mov v4.s[3], w11
+; CHECK-GI-NEXT: mov v21.s[1], w9
+; CHECK-GI-NEXT: ldrsb w9, [x0, #6]
; CHECK-GI-NEXT: mov v19.s[1], w13
-; CHECK-GI-NEXT: smov w13, v1.b[3]
-; CHECK-GI-NEXT: mov v23.s[2], wzr
-; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: smov w11, v0.b[15]
-; CHECK-GI-NEXT: mov v20.s[2], w10
-; CHECK-GI-NEXT: ldrsb w10, [x0, #2]
-; CHECK-GI-NEXT: mov v22.s[1], wzr
-; CHECK-GI-NEXT: mov v17.s[3], w12
+; CHECK-GI-NEXT: ldrsb w13, [x0, #1]
+; CHECK-GI-NEXT: mov v20.s[2], w8
+; CHECK-GI-NEXT: smov w8, v1.b[10]
+; CHECK-GI-NEXT: mov v18.s[3], w12
; CHECK-GI-NEXT: ldrsb w12, [x0, #7]
-; CHECK-GI-NEXT: mov v7.s[3], w8
-; CHECK-GI-NEXT: ldrsb w8, [x0, #3]
-; CHECK-GI-NEXT: mov v19.s[2], w9
-; CHECK-GI-NEXT: smov w9, v0.b[3]
-; CHECK-GI-NEXT: mov v18.s[3], w11
-; CHECK-GI-NEXT: smov w11, v0.b[11]
-; CHECK-GI-NEXT: mov v21.s[2], w10
+; CHECK-GI-NEXT: mov v21.s[2], w9
+; CHECK-GI-NEXT: smov w9, v2.b[11]
+; CHECK-GI-NEXT: fmov s2, w10
+; CHECK-GI-NEXT: ldrsb w10, [x0, #2]
+; CHECK-GI-NEXT: mov v22.s[1], w13
+; CHECK-GI-NEXT: smov w13, v1.b[15]
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v19.s[2], w8
+; CHECK-GI-NEXT: smov w8, v1.b[3]
+; CHECK-GI-NEXT: mov v21.s[3], w12
+; CHECK-GI-NEXT: mov v6.s[3], w9
+; CHECK-GI-NEXT: ldrsb w9, [x0, #3]
+; CHECK-GI-NEXT: mov v20.s[3], w13
+; CHECK-GI-NEXT: smov w13, v1.b[11]
+; CHECK-GI-NEXT: mov v22.s[2], w10
; CHECK-GI-NEXT: ldrsb w10, [x1, #3]
-; CHECK-GI-NEXT: mov v20.s[3], w12
-; CHECK-GI-NEXT: mov v22.s[2], wzr
-; CHECK-GI-NEXT: mov v6.s[3], w13
-; CHECK-GI-NEXT: mul v0.4s, v2.4s, v17.4s
-; CHECK-GI-NEXT: mov v23.s[3], wzr
+; CHECK-GI-NEXT: mul v1.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v17.s[3], w8
; CHECK-GI-NEXT: mov v3.s[3], w10
-; CHECK-GI-NEXT: mov v16.s[3], w9
-; CHECK-GI-NEXT: mov v19.s[3], w11
-; CHECK-GI-NEXT: mul v1.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT: mov v21.s[3], w8
-; CHECK-GI-NEXT: mul v2.4s, v5.4s, v20.4s
-; CHECK-GI-NEXT: mov v22.s[3], wzr
-; CHECK-GI-NEXT: mla v0.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT: mla v1.4s, v7.4s, v19.4s
-; CHECK-GI-NEXT: mla v2.4s, v3.4s, v21.4s
-; CHECK-GI-NEXT: add v3.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mul v5.4s, v5.4s, v21.4s
+; CHECK-GI-NEXT: mov v19.s[3], w13
+; CHECK-GI-NEXT: mul v7.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT: mov v22.s[3], w9
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mla v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT: mla v7.4s, v6.4s, v19.4s
+; CHECK-GI-NEXT: mla v5.4s, v3.4s, v22.4s
+; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v7.4s
+; CHECK-GI-NEXT: add v0.4s, v5.4s, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT: addv s0, v0.4s
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: add w0, w8, w2
@@ -2905,344 +2905,349 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25
;
; CHECK-GI-LABEL: test_sdot_v25i8_double:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str d10, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: stp d9, d8, [sp, #8] // 16-byte Folded Spill
-; CHECK-GI-NEXT: str x29, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT: .cfi_offset w29, -8
-; CHECK-GI-NEXT: .cfi_offset b8, -16
-; CHECK-GI-NEXT: .cfi_offset b9, -24
-; CHECK-GI-NEXT: .cfi_offset b10, -32
-; CHECK-GI-NEXT: ldr w8, [sp, #32]
-; CHECK-GI-NEXT: sxtb w9, w0
+; CHECK-GI-NEXT: stp d11, d10, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: str x29, [sp, #32] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT: .cfi_offset w29, -16
+; CHECK-GI-NEXT: .cfi_offset b8, -24
+; CHECK-GI-NEXT: .cfi_offset b9, -32
+; CHECK-GI-NEXT: .cfi_offset b10, -40
+; CHECK-GI-NEXT: .cfi_offset b11, -48
+; CHECK-GI-NEXT: sxtb w8, w0
; CHECK-GI-NEXT: sxtb w10, w4
-; CHECK-GI-NEXT: sxtb w11, w5
-; CHECK-GI-NEXT: sxtb w12, w3
-; CHECK-GI-NEXT: sxtb w13, w7
-; CHECK-GI-NEXT: mov v0.s[0], w9
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v2.s[0], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #40]
-; CHECK-GI-NEXT: sxtb w10, w1
-; CHECK-GI-NEXT: ldr w14, [sp, #152]
-; CHECK-GI-NEXT: mov v1.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #64]
-; CHECK-GI-NEXT: mov v9.s[0], wzr
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr x29, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.s[1], w10
-; CHECK-GI-NEXT: mov v2.s[1], w11
+; CHECK-GI-NEXT: sxtb w9, w1
+; CHECK-GI-NEXT: sxtb w11, w2
+; CHECK-GI-NEXT: sxtb w13, w6
+; CHECK-GI-NEXT: ldr w12, [sp, #72]
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #48]
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #80]
+; CHECK-GI-NEXT: ldr w14, [sp, #128]
+; CHECK-GI-NEXT: ldr w15, [sp, #152]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w10, w2
-; CHECK-GI-NEXT: sxtb w11, w6
-; CHECK-GI-NEXT: mov v1.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #48]
-; CHECK-GI-NEXT: mov v3.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #72]
-; CHECK-GI-NEXT: mov v9.s[1], wzr
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v0.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #56]
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: mov v2.s[1], w9
+; CHECK-GI-NEXT: sxtb w9, w5
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: ldr x29, [sp, #32] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v4.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #56]
+; CHECK-GI-NEXT: fmov s5, w10
; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: sxtb w10, w3
+; CHECK-GI-NEXT: mov v1.s[1], wzr
; CHECK-GI-NEXT: mov v2.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #80]
-; CHECK-GI-NEXT: mov v1.s[2], w9
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: ldr w11, [sp, #64]
+; CHECK-GI-NEXT: mov v5.s[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], w9
; CHECK-GI-NEXT: ldr w9, [sp, #96]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v3.s[1], w8
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: mov v0.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #128]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v2.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #168]
+; CHECK-GI-NEXT: mov v4.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #120]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v1.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #104]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v4.s[0], w9
+; CHECK-GI-NEXT: mov v2.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #112]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
; CHECK-GI-NEXT: mov v3.s[2], w11
-; CHECK-GI-NEXT: sxtb w11, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #136]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #112]
-; CHECK-GI-NEXT: ldr w12, [sp, #176]
-; CHECK-GI-NEXT: mov v6.s[0], w11
+; CHECK-GI-NEXT: sxtb w11, w10
+; CHECK-GI-NEXT: mov v5.s[2], w9
+; CHECK-GI-NEXT: sxtb w9, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #144]
+; CHECK-GI-NEXT: ldr w10, [sp, #136]
+; CHECK-GI-NEXT: fmov s6, w11
+; CHECK-GI-NEXT: sxtb w11, w7
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w11, [sp, #120]
-; CHECK-GI-NEXT: mov v9.s[2], wzr
-; CHECK-GI-NEXT: mov v4.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #200]
-; CHECK-GI-NEXT: mov v3.s[3], w8
-; CHECK-GI-NEXT: sxtb w8, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #144]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v5.s[1], w13
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w13, [sp, #184]
-; CHECK-GI-NEXT: mov v6.s[1], w8
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: mov v4.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #208]
-; CHECK-GI-NEXT: mov v7.s[0], w10
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v5.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #184]
+; CHECK-GI-NEXT: mov v4.s[3], w11
+; CHECK-GI-NEXT: mov v6.s[1], w9
+; CHECK-GI-NEXT: fmov s7, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #216]
+; CHECK-GI-NEXT: sxtb w9, w12
+; CHECK-GI-NEXT: sxtb w12, w14
+; CHECK-GI-NEXT: sxtb w14, w15
+; CHECK-GI-NEXT: sxtb w8, w8
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: sxtb w10, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #192]
-; CHECK-GI-NEXT: mov v5.s[2], w12
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w12, [sp, #232]
-; CHECK-GI-NEXT: mov v6.s[2], w13
-; CHECK-GI-NEXT: sxtb w13, w14
+; CHECK-GI-NEXT: ldr w11, [sp, #160]
+; CHECK-GI-NEXT: mov v7.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #224]
+; CHECK-GI-NEXT: mov v3.s[3], w9
+; CHECK-GI-NEXT: mov v6.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #192]
+; CHECK-GI-NEXT: fmov s16, w8
+; CHECK-GI-NEXT: fmov s18, w13
+; CHECK-GI-NEXT: sxtb w14, w14
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v4.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #216]
-; CHECK-GI-NEXT: mov v7.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #264]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w14, [sp, #368]
-; CHECK-GI-NEXT: mov v5.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #296]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v6.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #240]
-; CHECK-GI-NEXT: mov v16.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #224]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v7.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #272]
-; CHECK-GI-NEXT: mov v18.s[0], w9
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #304]
+; CHECK-GI-NEXT: ldr w9, [sp, #168]
+; CHECK-GI-NEXT: ldr w13, [sp, #208]
+; CHECK-GI-NEXT: mov v7.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #256]
+; CHECK-GI-NEXT: ldr w8, [sp, #176]
+; CHECK-GI-NEXT: mov v16.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #200]
+; CHECK-GI-NEXT: mov v18.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #232]
+; CHECK-GI-NEXT: mov v6.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #248]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[0], w13
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w13, [sp, #248]
-; CHECK-GI-NEXT: mov v16.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #328]
+; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v7.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #280]
-; CHECK-GI-NEXT: mov v18.s[1], w10
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w10, [sp, #312]
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v17.s[1], w9
+; CHECK-GI-NEXT: mov v16.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #240]
+; CHECK-GI-NEXT: mov v7.s[3], w9
+; CHECK-GI-NEXT: mov v18.s[2], w14
+; CHECK-GI-NEXT: fmov s17, w10
+; CHECK-GI-NEXT: ldr w14, [sp, #264]
; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w9, [sp, #288]
+; CHECK-GI-NEXT: ldr w10, [sp, #272]
+; CHECK-GI-NEXT: sxtb w14, w14
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v16.s[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #336]
+; CHECK-GI-NEXT: ldr w15, [sp, #392]
+; CHECK-GI-NEXT: mov v17.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #280]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v18.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #312]
+; CHECK-GI-NEXT: mov v16.s[3], w13
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w13, [sp, #296]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v19.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #288]
-; CHECK-GI-NEXT: mov v18.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #320]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: mov v17.s[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #320]
+; CHECK-GI-NEXT: fmov s20, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #344]
+; CHECK-GI-NEXT: fmov s19, w12
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w9, [sp, #256]
-; CHECK-GI-NEXT: mov v17.s[2], w10
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: ldr w12, [sp, #304]
+; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w10, [sp, #344]
+; CHECK-GI-NEXT: mov v20.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #352]
+; CHECK-GI-NEXT: mov v19.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #328]
; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: fmov s21, w11
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v7.4s
-; CHECK-GI-NEXT: mov v19.s[1], w13
-; CHECK-GI-NEXT: mov v18.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #360]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w13, [sp, #400]
-; CHECK-GI-NEXT: mov v16.s[3], w9
-; CHECK-GI-NEXT: mov v17.s[3], w12
-; CHECK-GI-NEXT: sxtb w12, w14
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #352]
-; CHECK-GI-NEXT: mov v9.s[3], wzr
-; CHECK-GI-NEXT: mla v2.4s, v0.4s, v6.4s
-; CHECK-GI-NEXT: mov v19.s[2], w10
+; CHECK-GI-NEXT: ldr w11, [sp, #336]
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: mov v17.s[3], w10
; CHECK-GI-NEXT: ldr w10, [sp, #376]
-; CHECK-GI-NEXT: mov v20.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #432]
-; CHECK-GI-NEXT: mul w8, w8, w11
-; CHECK-GI-NEXT: sxtb w11, w13
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w13, [sp, #384]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v23.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #408]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v20.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #440]
-; CHECK-GI-NEXT: mov v21.s[0], w8
+; CHECK-GI-NEXT: mov v20.s[2], w13
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v22.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #464]
-; CHECK-GI-NEXT: sxtb w8, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #416]
+; CHECK-GI-NEXT: ldr w13, [sp, #368]
+; CHECK-GI-NEXT: mov v21.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #360]
+; CHECK-GI-NEXT: mov v19.s[2], w14
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v23.s[1], w11
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v19.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #392]
-; CHECK-GI-NEXT: mov v20.s[2], w8
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v22.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #472]
-; CHECK-GI-NEXT: mov v24.s[0], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #424]
-; CHECK-GI-NEXT: ldr w8, [sp, #448]
+; CHECK-GI-NEXT: ldr w14, [sp, #384]
+; CHECK-GI-NEXT: mla v4.4s, v2.4s, v16.4s
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v23.s[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #496]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v20.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: mov v20.s[3], w12
+; CHECK-GI-NEXT: sxtb w12, w13
+; CHECK-GI-NEXT: mul w10, w8, w10
+; CHECK-GI-NEXT: mov v21.s[2], w9
+; CHECK-GI-NEXT: mov v19.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #416]
+; CHECK-GI-NEXT: sxtb w13, w14
+; CHECK-GI-NEXT: sxtb w14, w15
+; CHECK-GI-NEXT: ldr w9, [sp, #400]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v24.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #528]
-; CHECK-GI-NEXT: ldr w12, [sp, #456]
-; CHECK-GI-NEXT: mov v22.s[2], w8
-; CHECK-GI-NEXT: mov v23.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #504]
+; CHECK-GI-NEXT: fmov s22, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #432]
+; CHECK-GI-NEXT: fmov s23, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #448]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v25.s[0], w13
+; CHECK-GI-NEXT: mov v21.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #424]
+; CHECK-GI-NEXT: fmov s25, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #480]
+; CHECK-GI-NEXT: sxtb w13, w13
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w8, [sp, #488]
; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v23.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #456]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v24.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #536]
-; CHECK-GI-NEXT: mov v26.s[0], w10
-; CHECK-GI-NEXT: ldr w13, [sp, #568]
-; CHECK-GI-NEXT: mov v22.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #512]
+; CHECK-GI-NEXT: fmov s24, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #440]
+; CHECK-GI-NEXT: mov v25.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #488]
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: fmov s26, w11
+; CHECK-GI-NEXT: ldr w15, [sp, #504]
+; CHECK-GI-NEXT: ldr w11, [sp, #472]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v24.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #464]
+; CHECK-GI-NEXT: mov v23.s[2], w9
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w8, [sp, #408]
+; CHECK-GI-NEXT: mov v26.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #496]
+; CHECK-GI-NEXT: mov v25.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #512]
+; CHECK-GI-NEXT: sxtb w9, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #520]
+; CHECK-GI-NEXT: sxtb w12, w12
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v25.s[1], w11
+; CHECK-GI-NEXT: mov v22.s[1], wzr
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v24.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #528]
+; CHECK-GI-NEXT: mov v26.s[2], w12
+; CHECK-GI-NEXT: sxtb w12, w13
+; CHECK-GI-NEXT: sxtb w13, w15
+; CHECK-GI-NEXT: fmov s27, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #584]
+; CHECK-GI-NEXT: ldr w15, [sp, #552]
+; CHECK-GI-NEXT: mov v25.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #544]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: sxtb w10, w13
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v24.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #560]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v24.s[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #544]
-; CHECK-GI-NEXT: mov v26.s[1], w9
-; CHECK-GI-NEXT: ldr w13, [sp, #520]
-; CHECK-GI-NEXT: ldr w11, [sp, #576]
-; CHECK-GI-NEXT: mov v27.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #600]
+; CHECK-GI-NEXT: mov v26.s[3], w13
+; CHECK-GI-NEXT: sxtb w13, w14
+; CHECK-GI-NEXT: sxtb w14, w15
+; CHECK-GI-NEXT: fmov s29, w10
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: fmov s28, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #616]
+; CHECK-GI-NEXT: mov v27.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #592]
+; CHECK-GI-NEXT: ldr w15, [sp, #568]
+; CHECK-GI-NEXT: mov v23.s[3], w8
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w8, [sp, #536]
+; CHECK-GI-NEXT: ldr w10, [sp, #576]
+; CHECK-GI-NEXT: mov v28.s[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #624]
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: fmov s30, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #600]
+; CHECK-GI-NEXT: mov v27.s[2], w9
+; CHECK-GI-NEXT: mov v29.s[1], w13
+; CHECK-GI-NEXT: sxtb w13, w14
+; CHECK-GI-NEXT: sxtb w14, w15
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w9, [sp, #608]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v25.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #584]
-; CHECK-GI-NEXT: sxtb w9, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #552]
+; CHECK-GI-NEXT: mov v30.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #632]
+; CHECK-GI-NEXT: mov v28.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #640]
+; CHECK-GI-NEXT: sxtb w9, w9
; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v26.s[2], w8
-; CHECK-GI-NEXT: sxtb w8, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #664]
-; CHECK-GI-NEXT: mov v27.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #608]
-; CHECK-GI-NEXT: mov v28.s[0], w10
-; CHECK-GI-NEXT: mov v25.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #592]
+; CHECK-GI-NEXT: mov v29.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #648]
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #664]
+; CHECK-GI-NEXT: mov v30.s[2], w13
+; CHECK-GI-NEXT: mov v28.s[3], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #680]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #560]
-; CHECK-GI-NEXT: mov v26.s[3], w11
-; CHECK-GI-NEXT: sxtb w11, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #672]
-; CHECK-GI-NEXT: mov v30.s[0], w12
-; CHECK-GI-NEXT: mov v27.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #616]
-; CHECK-GI-NEXT: mov v28.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #632]
-; CHECK-GI-NEXT: ldr w12, [sp, #728]
-; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #656]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: mov v29.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #688]
+; CHECK-GI-NEXT: fmov s31, w12
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: ldr w12, [sp, #752]
+; CHECK-GI-NEXT: mov v30.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #744]
+; CHECK-GI-NEXT: fmov s8, w14
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: ldr w14, [sp, #712]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v21.s[1], wzr
-; CHECK-GI-NEXT: mov v30.s[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #760]
-; CHECK-GI-NEXT: mov v27.s[3], w11
-; CHECK-GI-NEXT: mov v28.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #696]
-; CHECK-GI-NEXT: mov v29.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #736]
-; CHECK-GI-NEXT: mov v8.s[0], w12
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: mov v31.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #696]
+; CHECK-GI-NEXT: mov v8.s[1], w9
+; CHECK-GI-NEXT: sxtb w14, w14
+; CHECK-GI-NEXT: ldr w9, [sp, #720]
+; CHECK-GI-NEXT: fmov s9, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #776]
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w11, [sp, #640]
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w12, [sp, #680]
+; CHECK-GI-NEXT: fmov s10, w14
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mul w10, w10, w13
-; CHECK-GI-NEXT: mov v21.s[2], wzr
+; CHECK-GI-NEXT: mov v22.s[2], wzr
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v31.s[0], w8
+; CHECK-GI-NEXT: mov v31.s[2], w8
; CHECK-GI-NEXT: ldr w8, [sp, #704]
-; CHECK-GI-NEXT: mov v8.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #744]
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v18.4s
-; CHECK-GI-NEXT: mov v10.s[0], w10
-; CHECK-GI-NEXT: mov v29.s[1], w11
+; CHECK-GI-NEXT: mov v9.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #760]
+; CHECK-GI-NEXT: mov v8.s[2], w13
+; CHECK-GI-NEXT: mul w10, w10, w11
+; CHECK-GI-NEXT: mov v10.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #728]
; CHECK-GI-NEXT: sxtb w11, w12
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w12, [sp, #624]
-; CHECK-GI-NEXT: mov v30.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #648]
-; CHECK-GI-NEXT: ldr w10, [sp, #688]
-; CHECK-GI-NEXT: mov v31.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #712]
-; CHECK-GI-NEXT: mov v8.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #752]
-; CHECK-GI-NEXT: mov v10.s[1], wzr
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: mul v5.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s
+; CHECK-GI-NEXT: mul v18.4s, v25.4s, v30.4s
+; CHECK-GI-NEXT: mov v22.s[3], wzr
+; CHECK-GI-NEXT: fmov s11, w10
+; CHECK-GI-NEXT: mov v9.s[2], w11
+; CHECK-GI-NEXT: ldr w10, [sp, #768]
+; CHECK-GI-NEXT: mov v8.s[3], w8
+; CHECK-GI-NEXT: sxtb w8, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #672]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mla v5.4s, v3.4s, v17.4s
+; CHECK-GI-NEXT: mov v11.s[1], wzr
+; CHECK-GI-NEXT: mov v10.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #736]
+; CHECK-GI-NEXT: mov v9.s[3], w10
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v28.s[3], w12
-; CHECK-GI-NEXT: mul v5.4s, v5.4s, v19.4s
-; CHECK-GI-NEXT: mov v29.s[2], w11
-; CHECK-GI-NEXT: mov v30.s[3], w10
-; CHECK-GI-NEXT: mov v31.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #656]
-; CHECK-GI-NEXT: mov v8.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #720]
-; CHECK-GI-NEXT: mov v10.s[2], wzr
-; CHECK-GI-NEXT: mov v21.s[3], wzr
-; CHECK-GI-NEXT: mla v3.4s, v1.4s, v16.4s
+; CHECK-GI-NEXT: mla v7.4s, v6.4s, v19.4s
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mul v7.4s, v23.4s, v28.4s
-; CHECK-GI-NEXT: mul v18.4s, v24.4s, v30.4s
-; CHECK-GI-NEXT: mla v5.4s, v4.4s, v17.4s
-; CHECK-GI-NEXT: mul v19.4s, v26.4s, v8.4s
-; CHECK-GI-NEXT: mov v29.s[3], w8
+; CHECK-GI-NEXT: mul v20.4s, v26.4s, v8.4s
+; CHECK-GI-NEXT: mla v18.4s, v23.4s, v29.4s
; CHECK-GI-NEXT: mov v31.s[3], w9
-; CHECK-GI-NEXT: mov v10.s[3], wzr
-; CHECK-GI-NEXT: add v0.4s, v21.4s, v9.4s
-; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: mla v7.4s, v20.4s, v27.4s
-; CHECK-GI-NEXT: mla v18.4s, v22.4s, v29.4s
-; CHECK-GI-NEXT: mla v19.4s, v25.4s, v31.4s
-; CHECK-GI-NEXT: add v0.4s, v5.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v10.4s, v9.4s
-; CHECK-GI-NEXT: ldp d9, d8, [sp, #8] // 16-byte Folded Reload
-; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
-; CHECK-GI-NEXT: add v3.4s, v7.4s, v18.4s
-; CHECK-GI-NEXT: add v1.4s, v19.4s, v1.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add v1.4s, v22.4s, v1.4s
+; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mov v11.s[2], wzr
+; CHECK-GI-NEXT: mov v10.s[3], w8
+; CHECK-GI-NEXT: mul v21.4s, v28.4s, v9.4s
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-GI-NEXT: mla v20.4s, v24.4s, v31.4s
+; CHECK-GI-NEXT: mov v11.s[3], wzr
+; CHECK-GI-NEXT: mla v21.4s, v27.4s, v10.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: add v3.4s, v18.4s, v20.4s
+; CHECK-GI-NEXT: add v0.4s, v11.4s, v0.4s
; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add v0.4s, v21.4s, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ldr d10, [sp], #32 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldp d11, d10, [sp], #48 // 16-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%az = sext <25 x i8> %a to <25 x i32>
@@ -3386,175 +3391,179 @@ define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
; CHECK-GI-NEXT: sxtb w8, w0
-; CHECK-GI-NEXT: sxtb w9, w4
-; CHECK-GI-NEXT: ldr w10, [sp, #48]
-; CHECK-GI-NEXT: sxtb w11, w5
-; CHECK-GI-NEXT: sxtb w12, w6
-; CHECK-GI-NEXT: sxtb w13, w7
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #16]
-; CHECK-GI-NEXT: mov v1.s[0], w9
; CHECK-GI-NEXT: sxtb w9, w1
-; CHECK-GI-NEXT: ldr w14, [sp, #104]
-; CHECK-GI-NEXT: mov v20.s[0], wzr
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v0.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #80]
-; CHECK-GI-NEXT: mov v2.s[0], w8
-; CHECK-GI-NEXT: sxtb w8, w10
-; CHECK-GI-NEXT: mov v1.s[1], w11
-; CHECK-GI-NEXT: ldr w10, [sp, #24]
-; CHECK-GI-NEXT: sxtb w11, w2
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v20.s[1], wzr
-; CHECK-GI-NEXT: mov v3.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #56]
-; CHECK-GI-NEXT: mov v4.s[0], w9
-; CHECK-GI-NEXT: sxtb w9, w10
-; CHECK-GI-NEXT: mov v0.s[2], w11
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w10, [sp, #32]
-; CHECK-GI-NEXT: mov v1.s[2], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #48]
+; CHECK-GI-NEXT: sxtb w11, w6
; CHECK-GI-NEXT: sxtb w12, w3
+; CHECK-GI-NEXT: sxtb w13, w7
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: sxtb w8, w4
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: ldr w14, [sp, #448]
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: sxtb w8, w2
+; CHECK-GI-NEXT: fmov s5, w10
; CHECK-GI-NEXT: mov v2.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #112]
-; CHECK-GI-NEXT: mov v3.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: sxtb w9, w5
+; CHECK-GI-NEXT: ldr w10, [sp, #80]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #16]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v0.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #64]
+; CHECK-GI-NEXT: mov v2.s[2], w8
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: ldr w8, [sp, #24]
+; CHECK-GI-NEXT: fmov s6, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #64]
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v3.s[2], w11
+; CHECK-GI-NEXT: fmov s4, w9
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w11, [sp, #40]
-; CHECK-GI-NEXT: mov v1.s[3], w13
-; CHECK-GI-NEXT: mov v2.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #120]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v5.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #96]
+; CHECK-GI-NEXT: ldr w9, [sp, #56]
+; CHECK-GI-NEXT: ldr w11, [sp, #32]
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v2.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #88]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: ldr w13, [sp, #72]
-; CHECK-GI-NEXT: sxtb w8, w10
-; CHECK-GI-NEXT: mov v3.s[2], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #352]
+; CHECK-GI-NEXT: ldr w8, [sp, #120]
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v3.s[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #112]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: sxtb w8, w8
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w12, w13
-; CHECK-GI-NEXT: sxtb w13, w14
-; CHECK-GI-NEXT: mov v20.s[2], wzr
-; CHECK-GI-NEXT: mov v4.s[2], w9
-; CHECK-GI-NEXT: sxtb w9, w10
-; CHECK-GI-NEXT: mov v2.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #128]
-; CHECK-GI-NEXT: mov v5.s[1], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #384]
-; CHECK-GI-NEXT: mov v3.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #360]
-; CHECK-GI-NEXT: mov v6.s[0], w9
+; CHECK-GI-NEXT: mov v5.s[1], w9
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v6.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #128]
+; CHECK-GI-NEXT: mov v4.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #104]
+; CHECK-GI-NEXT: ldr w9, [sp, #40]
+; CHECK-GI-NEXT: fmov s7, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #96]
+; CHECK-GI-NEXT: sxtb w12, w12
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #144]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v4.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #392]
-; CHECK-GI-NEXT: mov v5.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #368]
-; CHECK-GI-NEXT: mov v7.s[0], w10
+; CHECK-GI-NEXT: mov v5.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #136]
+; CHECK-GI-NEXT: sxtb w13, w13
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v6.s[1], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #416]
-; CHECK-GI-NEXT: sxtb w12, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #448]
-; CHECK-GI-NEXT: ldr w8, [sp, #136]
-; CHECK-GI-NEXT: mov v16.s[0], w9
-; CHECK-GI-NEXT: sxtb w9, w11
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v7.s[1], w8
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w11, [sp, #400]
-; CHECK-GI-NEXT: mov v7.s[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #480]
-; CHECK-GI-NEXT: mov v6.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #424]
-; CHECK-GI-NEXT: mov v17.s[0], w10
+; CHECK-GI-NEXT: ldr w8, [sp, #72]
+; CHECK-GI-NEXT: mov v6.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #144]
+; CHECK-GI-NEXT: mov v4.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #360]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v0.s[3], wzr
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: mov v7.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #352]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #456]
-; CHECK-GI-NEXT: mov v16.s[1], wzr
-; CHECK-GI-NEXT: mov v18.s[0], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #488]
-; CHECK-GI-NEXT: mov v7.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #512]
-; CHECK-GI-NEXT: mov v19.s[0], w12
-; CHECK-GI-NEXT: mov v17.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #544]
+; CHECK-GI-NEXT: mov v6.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #384]
+; CHECK-GI-NEXT: fmov s16, w13
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w13, [sp, #416]
+; CHECK-GI-NEXT: mov v5.s[3], w8
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w8, [sp, #368]
+; CHECK-GI-NEXT: mov v7.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #392]
+; CHECK-GI-NEXT: fmov s17, w12
+; CHECK-GI-NEXT: fmov s18, w11
+; CHECK-GI-NEXT: ldr w12, [sp, #424]
+; CHECK-GI-NEXT: ldr w11, [sp, #456]
; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: sxtb w8, w8
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w14, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #376]
-; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v17.s[1], w9
+; CHECK-GI-NEXT: mov v16.s[1], wzr
; CHECK-GI-NEXT: mov v18.s[1], w10
+; CHECK-GI-NEXT: sxtb w10, w12
+; CHECK-GI-NEXT: sxtb w12, w14
+; CHECK-GI-NEXT: fmov s19, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #400]
+; CHECK-GI-NEXT: ldr w9, [sp, #376]
+; CHECK-GI-NEXT: fmov s20, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #464]
+; CHECK-GI-NEXT: add v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v17.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #488]
+; CHECK-GI-NEXT: mov v19.s[1], w10
; CHECK-GI-NEXT: ldr w10, [sp, #432]
-; CHECK-GI-NEXT: mov v19.s[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #520]
-; CHECK-GI-NEXT: mov v21.s[0], w11
-; CHECK-GI-NEXT: mov v22.s[0], w9
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v20.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #480]
+; CHECK-GI-NEXT: mov v18.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #512]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w11, [sp, #464]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w9, [sp, #496]
-; CHECK-GI-NEXT: ldr w12, [sp, #408]
-; CHECK-GI-NEXT: mov v17.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #528]
+; CHECK-GI-NEXT: sxtb w8, w8
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v21.s[1], w13
+; CHECK-GI-NEXT: mov v16.s[2], wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w13, [sp, #440]
-; CHECK-GI-NEXT: mov v22.s[1], wzr
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v19.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #520]
+; CHECK-GI-NEXT: fmov s21, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #544]
+; CHECK-GI-NEXT: mov v20.s[2], w12
+; CHECK-GI-NEXT: fmov s22, w13
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #472]
-; CHECK-GI-NEXT: mov v19.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #504]
-; CHECK-GI-NEXT: mov v16.s[2], wzr
+; CHECK-GI-NEXT: ldr w13, [sp, #528]
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w12, [sp, #408]
+; CHECK-GI-NEXT: mov v17.s[3], w9
+; CHECK-GI-NEXT: mov v21.s[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #496]
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: fmov s23, w11
+; CHECK-GI-NEXT: mov v22.s[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #440]
; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: ldr w11, [sp, #472]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v21.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #536]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v22.s[2], wzr
-; CHECK-GI-NEXT: mov v6.s[3], w8
-; CHECK-GI-NEXT: sxtb w8, w11
-; CHECK-GI-NEXT: sxtb w9, w9
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v5.s[3], w14
-; CHECK-GI-NEXT: mov v7.s[3], w12
-; CHECK-GI-NEXT: mov v17.s[3], w13
-; CHECK-GI-NEXT: mov v18.s[3], w8
; CHECK-GI-NEXT: mov v16.s[3], wzr
-; CHECK-GI-NEXT: mov v20.s[3], wzr
-; CHECK-GI-NEXT: mov v19.s[3], w9
-; CHECK-GI-NEXT: mov v21.s[3], w10
-; CHECK-GI-NEXT: mov v22.s[3], wzr
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT: add v4.4s, v17.4s, v18.4s
-; CHECK-GI-NEXT: add v6.4s, v16.4s, v20.4s
-; CHECK-GI-NEXT: add v5.4s, v19.4s, v21.4s
-; CHECK-GI-NEXT: add v7.4s, v22.4s, v20.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v6.4s
-; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: add v3.4s, v5.4s, v7.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: add v4.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mov v23.s[1], wzr
+; CHECK-GI-NEXT: mov v21.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #504]
+; CHECK-GI-NEXT: mov v22.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #536]
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v18.s[3], w12
+; CHECK-GI-NEXT: mov v19.s[3], w10
+; CHECK-GI-NEXT: sxtb w9, w13
+; CHECK-GI-NEXT: mov v20.s[3], w11
+; CHECK-GI-NEXT: add v1.4s, v16.4s, v1.4s
+; CHECK-GI-NEXT: mov v23.s[2], wzr
+; CHECK-GI-NEXT: mov v21.s[3], w8
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: mov v22.s[3], w9
+; CHECK-GI-NEXT: add v5.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: add v1.4s, v4.4s, v1.4s
+; CHECK-GI-NEXT: add v6.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT: mov v23.s[3], wzr
+; CHECK-GI-NEXT: add v7.4s, v21.4s, v22.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: add v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: add v0.4s, v23.4s, v0.4s
; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add v0.4s, v7.4s, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
@@ -3920,167 +3929,197 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_udot_v33i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldp q19, q4, [x1]
-; CHECK-GI-NEXT: mov v25.s[0], wzr
-; CHECK-GI-NEXT: ldp q7, q5, [x0]
-; CHECK-GI-NEXT: umov w8, v19.b[0]
-; CHECK-GI-NEXT: umov w9, v19.b[4]
-; CHECK-GI-NEXT: umov w10, v19.b[8]
-; CHECK-GI-NEXT: umov w11, v19.b[1]
-; CHECK-GI-NEXT: umov w12, v19.b[6]
-; CHECK-GI-NEXT: umov w13, v19.b[12]
-; CHECK-GI-NEXT: umov w14, v4.b[0]
-; CHECK-GI-NEXT: umov w15, v4.b[4]
-; CHECK-GI-NEXT: umov w16, v4.b[12]
-; CHECK-GI-NEXT: mov v25.s[1], wzr
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: umov w8, v19.b[5]
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: umov w9, v19.b[9]
-; CHECK-GI-NEXT: mov v1.s[0], w10
-; CHECK-GI-NEXT: umov w10, v19.b[2]
-; CHECK-GI-NEXT: mov v6.s[0], w13
-; CHECK-GI-NEXT: umov w13, v19.b[3]
-; CHECK-GI-NEXT: mov v3.s[0], w14
-; CHECK-GI-NEXT: umov w14, v19.b[13]
-; CHECK-GI-NEXT: mov v16.s[0], w15
-; CHECK-GI-NEXT: umov w15, v4.b[8]
-; CHECK-GI-NEXT: mov v0.s[1], w11
-; CHECK-GI-NEXT: umov w11, v19.b[10]
-; CHECK-GI-NEXT: mov v2.s[1], w8
-; CHECK-GI-NEXT: ldrb w8, [x0, #32]
-; CHECK-GI-NEXT: mov v1.s[1], w9
-; CHECK-GI-NEXT: ldrb w9, [x1, #32]
-; CHECK-GI-NEXT: mov v17.s[0], w16
-; CHECK-GI-NEXT: umov w16, v19.b[14]
-; CHECK-GI-NEXT: mov v25.s[2], wzr
-; CHECK-GI-NEXT: mul w8, w9, w8
-; CHECK-GI-NEXT: mov v6.s[1], w14
-; CHECK-GI-NEXT: umov w14, v4.b[1]
-; CHECK-GI-NEXT: mov v0.s[2], w10
-; CHECK-GI-NEXT: umov w10, v19.b[7]
-; CHECK-GI-NEXT: mov v2.s[2], w12
-; CHECK-GI-NEXT: umov w12, v19.b[11]
-; CHECK-GI-NEXT: mov v1.s[2], w11
-; CHECK-GI-NEXT: umov w11, v4.b[5]
-; CHECK-GI-NEXT: mov v18.s[0], w15
-; CHECK-GI-NEXT: umov w15, v19.b[15]
-; CHECK-GI-NEXT: umov w9, v5.b[2]
-; CHECK-GI-NEXT: mov v6.s[2], w16
-; CHECK-GI-NEXT: umov w16, v7.b[0]
-; CHECK-GI-NEXT: mov v3.s[1], w14
-; CHECK-GI-NEXT: mov v0.s[3], w13
-; CHECK-GI-NEXT: umov w13, v7.b[4]
-; CHECK-GI-NEXT: mov v2.s[3], w10
-; CHECK-GI-NEXT: umov w10, v4.b[6]
-; CHECK-GI-NEXT: mov v1.s[3], w12
-; CHECK-GI-NEXT: umov w12, v4.b[13]
-; CHECK-GI-NEXT: mov v16.s[1], w11
-; CHECK-GI-NEXT: umov w11, v4.b[9]
-; CHECK-GI-NEXT: umov w14, v7.b[5]
-; CHECK-GI-NEXT: mov v19.s[0], w16
-; CHECK-GI-NEXT: umov w16, v7.b[1]
-; CHECK-GI-NEXT: mov v6.s[3], w15
-; CHECK-GI-NEXT: mov v20.s[0], w13
-; CHECK-GI-NEXT: umov w13, v4.b[2]
-; CHECK-GI-NEXT: umov w15, v7.b[6]
-; CHECK-GI-NEXT: mov v17.s[1], w12
-; CHECK-GI-NEXT: umov w12, v4.b[14]
-; CHECK-GI-NEXT: mov v27.s[0], w8
-; CHECK-GI-NEXT: mov v16.s[2], w10
-; CHECK-GI-NEXT: umov w10, v4.b[7]
-; CHECK-GI-NEXT: mov v18.s[1], w11
-; CHECK-GI-NEXT: umov w11, v4.b[10]
-; CHECK-GI-NEXT: mov v19.s[1], w16
-; CHECK-GI-NEXT: umov w16, v5.b[4]
+; CHECK-GI-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: .cfi_offset b8, -16
+; CHECK-GI-NEXT: ldp q21, q25, [x1]
+; CHECK-GI-NEXT: fmov s5, wzr
+; CHECK-GI-NEXT: ldp q26, q22, [x0]
+; CHECK-GI-NEXT: fmov s6, wzr
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s3, wzr
+; CHECK-GI-NEXT: umov w8, v21.b[0]
+; CHECK-GI-NEXT: umov w9, v21.b[4]
+; CHECK-GI-NEXT: umov w10, v21.b[1]
+; CHECK-GI-NEXT: umov w13, v21.b[8]
+; CHECK-GI-NEXT: umov w11, v21.b[5]
+; CHECK-GI-NEXT: umov w14, v21.b[9]
+; CHECK-GI-NEXT: umov w15, v25.b[0]
+; CHECK-GI-NEXT: umov w12, v21.b[2]
+; CHECK-GI-NEXT: fmov s2, wzr
+; CHECK-GI-NEXT: fmov s4, wzr
+; CHECK-GI-NEXT: mov v5.s[1], wzr
+; CHECK-GI-NEXT: mov v6.s[1], wzr
+; CHECK-GI-NEXT: fmov s7, w8
+; CHECK-GI-NEXT: fmov s17, w9
+; CHECK-GI-NEXT: umov w8, v21.b[6]
+; CHECK-GI-NEXT: fmov s16, w13
+; CHECK-GI-NEXT: umov w9, v21.b[3]
+; CHECK-GI-NEXT: umov w13, v21.b[7]
+; CHECK-GI-NEXT: fmov s18, w15
+; CHECK-GI-NEXT: umov w15, v25.b[4]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v7.s[1], w10
+; CHECK-GI-NEXT: umov w10, v21.b[12]
+; CHECK-GI-NEXT: mov v17.s[1], w11
+; CHECK-GI-NEXT: umov w11, v21.b[13]
+; CHECK-GI-NEXT: mov v16.s[1], w14
+; CHECK-GI-NEXT: umov w14, v25.b[1]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], wzr
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: fmov s20, w15
+; CHECK-GI-NEXT: umov w15, v25.b[13]
+; CHECK-GI-NEXT: mov v4.s[1], wzr
+; CHECK-GI-NEXT: fmov s19, w10
+; CHECK-GI-NEXT: mov v7.s[2], w12
+; CHECK-GI-NEXT: umov w12, v21.b[10]
+; CHECK-GI-NEXT: mov v18.s[1], w14
+; CHECK-GI-NEXT: umov w14, v25.b[5]
+; CHECK-GI-NEXT: mov v17.s[2], w8
+; CHECK-GI-NEXT: umov w8, v21.b[11]
+; CHECK-GI-NEXT: umov w10, v21.b[14]
+; CHECK-GI-NEXT: mov v5.s[2], wzr
+; CHECK-GI-NEXT: mov v19.s[1], w11
+; CHECK-GI-NEXT: umov w11, v25.b[2]
+; CHECK-GI-NEXT: mov v6.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[2], w12
+; CHECK-GI-NEXT: umov w12, v25.b[8]
+; CHECK-GI-NEXT: mov v7.s[3], w9
; CHECK-GI-NEXT: mov v20.s[1], w14
-; CHECK-GI-NEXT: umov w14, v4.b[15]
-; CHECK-GI-NEXT: mov v3.s[2], w13
-; CHECK-GI-NEXT: mov v17.s[2], w12
-; CHECK-GI-NEXT: umov w12, v7.b[12]
-; CHECK-GI-NEXT: umov w13, v7.b[7]
-; CHECK-GI-NEXT: mov v16.s[3], w10
-; CHECK-GI-NEXT: umov w10, v7.b[8]
-; CHECK-GI-NEXT: umov w8, v7.b[3]
+; CHECK-GI-NEXT: umov w14, v21.b[15]
+; CHECK-GI-NEXT: umov w9, v25.b[9]
+; CHECK-GI-NEXT: mov v17.s[3], w13
+; CHECK-GI-NEXT: umov w13, v25.b[12]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: umov w11, v7.b[2]
-; CHECK-GI-NEXT: mov v23.s[0], w16
-; CHECK-GI-NEXT: mov v20.s[2], w15
-; CHECK-GI-NEXT: umov w15, v5.b[12]
-; CHECK-GI-NEXT: umov w16, v7.b[14]
-; CHECK-GI-NEXT: mov v17.s[3], w14
-; CHECK-GI-NEXT: umov w14, v7.b[13]
-; CHECK-GI-NEXT: mov v22.s[0], w12
-; CHECK-GI-NEXT: umov w12, v7.b[9]
-; CHECK-GI-NEXT: mov v21.s[0], w10
-; CHECK-GI-NEXT: umov w10, v4.b[3]
-; CHECK-GI-NEXT: mov v19.s[2], w11
-; CHECK-GI-NEXT: umov w11, v5.b[0]
-; CHECK-GI-NEXT: mov v27.s[1], wzr
-; CHECK-GI-NEXT: mov v20.s[3], w13
-; CHECK-GI-NEXT: umov w13, v5.b[5]
-; CHECK-GI-NEXT: mov v24.s[0], w15
-; CHECK-GI-NEXT: mov v22.s[1], w14
-; CHECK-GI-NEXT: umov w14, v5.b[8]
-; CHECK-GI-NEXT: umov w15, v4.b[11]
-; CHECK-GI-NEXT: mov v21.s[1], w12
-; CHECK-GI-NEXT: umov w12, v5.b[13]
-; CHECK-GI-NEXT: mov v25.s[3], wzr
-; CHECK-GI-NEXT: mov v4.s[0], w11
-; CHECK-GI-NEXT: umov w11, v5.b[1]
-; CHECK-GI-NEXT: mov v3.s[3], w10
-; CHECK-GI-NEXT: mov v23.s[1], w13
-; CHECK-GI-NEXT: umov w13, v5.b[6]
-; CHECK-GI-NEXT: mov v19.s[3], w8
-; CHECK-GI-NEXT: mov v22.s[2], w16
-; CHECK-GI-NEXT: umov w16, v5.b[9]
-; CHECK-GI-NEXT: mov v26.s[0], w14
+; CHECK-GI-NEXT: umov w11, v26.b[0]
+; CHECK-GI-NEXT: mov v19.s[2], w10
+; CHECK-GI-NEXT: fmov s21, w12
+; CHECK-GI-NEXT: umov w12, v26.b[1]
+; CHECK-GI-NEXT: mov v16.s[3], w8
+; CHECK-GI-NEXT: umov w8, v26.b[5]
+; CHECK-GI-NEXT: umov w10, v25.b[6]
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: fmov s23, w13
+; CHECK-GI-NEXT: umov w13, v25.b[3]
+; CHECK-GI-NEXT: mov v3.s[2], wzr
+; CHECK-GI-NEXT: fmov s24, w11
+; CHECK-GI-NEXT: mov v21.s[1], w9
+; CHECK-GI-NEXT: umov w9, v25.b[10]
+; CHECK-GI-NEXT: umov w11, v26.b[2]
+; CHECK-GI-NEXT: mov v19.s[3], w14
+; CHECK-GI-NEXT: umov w14, v26.b[13]
+; CHECK-GI-NEXT: mov v23.s[1], w15
+; CHECK-GI-NEXT: umov w15, v25.b[14]
+; CHECK-GI-NEXT: mov v20.s[2], w10
; CHECK-GI-NEXT: mov v24.s[1], w12
-; CHECK-GI-NEXT: umov w12, v5.b[14]
-; CHECK-GI-NEXT: umov w14, v7.b[10]
-; CHECK-GI-NEXT: mov v4.s[1], w11
-; CHECK-GI-NEXT: umov w11, v7.b[15]
-; CHECK-GI-NEXT: mov v18.s[3], w15
-; CHECK-GI-NEXT: mov v23.s[2], w13
-; CHECK-GI-NEXT: umov w13, v5.b[7]
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v20.4s
-; CHECK-GI-NEXT: mov v26.s[1], w16
-; CHECK-GI-NEXT: umov w16, v5.b[10]
-; CHECK-GI-NEXT: mov v27.s[2], wzr
-; CHECK-GI-NEXT: mov v24.s[2], w12
-; CHECK-GI-NEXT: umov w12, v5.b[15]
-; CHECK-GI-NEXT: mov v21.s[2], w14
-; CHECK-GI-NEXT: umov w14, v7.b[11]
-; CHECK-GI-NEXT: mov v4.s[2], w9
-; CHECK-GI-NEXT: umov w9, v5.b[3]
-; CHECK-GI-NEXT: mov v22.s[3], w11
-; CHECK-GI-NEXT: umov w11, v5.b[11]
-; CHECK-GI-NEXT: mov v23.s[3], w13
-; CHECK-GI-NEXT: mov v26.s[2], w16
-; CHECK-GI-NEXT: mla v2.4s, v0.4s, v19.4s
-; CHECK-GI-NEXT: mov v27.s[3], wzr
-; CHECK-GI-NEXT: mov v24.s[3], w12
-; CHECK-GI-NEXT: mov v21.s[3], w14
-; CHECK-GI-NEXT: mov v4.s[3], w9
-; CHECK-GI-NEXT: mul v5.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT: mul v6.4s, v16.4s, v23.4s
-; CHECK-GI-NEXT: add v16.4s, v25.4s, v25.4s
-; CHECK-GI-NEXT: mov v26.s[3], w11
-; CHECK-GI-NEXT: mul v7.4s, v17.4s, v24.4s
-; CHECK-GI-NEXT: add v0.4s, v25.4s, v16.4s
-; CHECK-GI-NEXT: mla v5.4s, v1.4s, v21.4s
-; CHECK-GI-NEXT: mla v6.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: add v3.4s, v16.4s, v16.4s
-; CHECK-GI-NEXT: mla v7.4s, v18.4s, v26.4s
-; CHECK-GI-NEXT: add v0.4s, v27.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: umov w12, v26.b[4]
+; CHECK-GI-NEXT: umov w10, v25.b[7]
+; CHECK-GI-NEXT: mov v21.s[2], w9
+; CHECK-GI-NEXT: umov w9, v25.b[11]
+; CHECK-GI-NEXT: mov v18.s[3], w13
+; CHECK-GI-NEXT: umov w13, v26.b[9]
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v4.s[2], wzr
+; CHECK-GI-NEXT: mov v23.s[2], w15
+; CHECK-GI-NEXT: umov w15, v25.b[15]
+; CHECK-GI-NEXT: mov v5.s[3], wzr
+; CHECK-GI-NEXT: fmov s27, w12
+; CHECK-GI-NEXT: mov v24.s[2], w11
+; CHECK-GI-NEXT: umov w11, v26.b[6]
+; CHECK-GI-NEXT: umov w12, v26.b[8]
+; CHECK-GI-NEXT: mov v21.s[3], w9
+; CHECK-GI-NEXT: umov w9, v26.b[12]
+; CHECK-GI-NEXT: mov v20.s[3], w10
+; CHECK-GI-NEXT: umov w10, v26.b[3]
+; CHECK-GI-NEXT: mov v6.s[3], wzr
+; CHECK-GI-NEXT: mov v27.s[1], w8
+; CHECK-GI-NEXT: mov v23.s[3], w15
+; CHECK-GI-NEXT: umov w15, v22.b[0]
+; CHECK-GI-NEXT: umov w8, v26.b[7]
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: fmov s25, w12
+; CHECK-GI-NEXT: fmov s29, w9
+; CHECK-GI-NEXT: umov w9, v22.b[5]
+; CHECK-GI-NEXT: mov v24.s[3], w10
+; CHECK-GI-NEXT: umov w10, v22.b[1]
+; CHECK-GI-NEXT: umov w12, v26.b[10]
+; CHECK-GI-NEXT: mov v27.s[2], w11
+; CHECK-GI-NEXT: umov w11, v22.b[4]
+; CHECK-GI-NEXT: fmov s28, w15
+; CHECK-GI-NEXT: mov v25.s[1], w13
+; CHECK-GI-NEXT: umov w13, v26.b[14]
+; CHECK-GI-NEXT: mov v29.s[1], w14
+; CHECK-GI-NEXT: umov w15, v22.b[12]
+; CHECK-GI-NEXT: umov w14, v22.b[2]
+; CHECK-GI-NEXT: mov v3.s[3], wzr
+; CHECK-GI-NEXT: mov v28.s[1], w10
+; CHECK-GI-NEXT: umov w10, v22.b[13]
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: fmov s30, w11
+; CHECK-GI-NEXT: umov w11, v22.b[6]
+; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: mov v25.s[2], w12
+; CHECK-GI-NEXT: mov v29.s[2], w13
+; CHECK-GI-NEXT: umov w13, v26.b[11]
+; CHECK-GI-NEXT: fmov s31, w15
+; CHECK-GI-NEXT: umov w15, v26.b[15]
+; CHECK-GI-NEXT: umov w12, v22.b[9]
+; CHECK-GI-NEXT: mov v30.s[1], w9
+; CHECK-GI-NEXT: umov w9, v22.b[8]
+; CHECK-GI-NEXT: mov v28.s[2], w14
+; CHECK-GI-NEXT: ldrb w14, [x1, #32]
+; CHECK-GI-NEXT: umov w8, v22.b[15]
+; CHECK-GI-NEXT: mul v17.4s, v17.4s, v27.4s
+; CHECK-GI-NEXT: mov v31.s[1], w10
+; CHECK-GI-NEXT: umov w10, v22.b[14]
+; CHECK-GI-NEXT: mov v25.s[3], w13
+; CHECK-GI-NEXT: ldrb w13, [x0, #32]
+; CHECK-GI-NEXT: mov v29.s[3], w15
+; CHECK-GI-NEXT: mov v4.s[3], wzr
+; CHECK-GI-NEXT: mov v30.s[2], w11
+; CHECK-GI-NEXT: fmov s26, w9
+; CHECK-GI-NEXT: umov w9, v22.b[7]
+; CHECK-GI-NEXT: umov w11, v22.b[3]
+; CHECK-GI-NEXT: add v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: mla v17.4s, v7.4s, v24.4s
+; CHECK-GI-NEXT: mov v31.s[2], w10
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: mov v26.s[1], w12
+; CHECK-GI-NEXT: umov w12, v22.b[10]
+; CHECK-GI-NEXT: mul v19.4s, v19.4s, v29.4s
+; CHECK-GI-NEXT: mov v30.s[3], w9
+; CHECK-GI-NEXT: mul w9, w14, w13
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: mov v28.s[3], w11
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT: mov v31.s[3], w8
+; CHECK-GI-NEXT: umov w8, v22.b[11]
+; CHECK-GI-NEXT: fmov s8, w9
+; CHECK-GI-NEXT: mov v26.s[2], w12
+; CHECK-GI-NEXT: mla v19.4s, v16.4s, v25.4s
+; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s
; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: mov v8.s[1], wzr
+; CHECK-GI-NEXT: mul v22.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT: mov v26.s[3], w8
+; CHECK-GI-NEXT: add v3.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT: mla v20.4s, v18.4s, v28.4s
+; CHECK-GI-NEXT: mov v8.s[2], wzr
+; CHECK-GI-NEXT: mla v22.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT: mov v8.s[3], wzr
+; CHECK-GI-NEXT: add v4.4s, v20.4s, v22.4s
+; CHECK-GI-NEXT: add v0.4s, v8.4s, v0.4s
+; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT: addv s0, v0.4s
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%0 = load <33 x i8>, ptr %a
@@ -4120,115 +4159,126 @@ define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {
;
; CHECK-GI-LABEL: test_udot_v33i8_nomla:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x27, [sp, #-80]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT: stp x20, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w19, -8
; CHECK-GI-NEXT: .cfi_offset w20, -16
-; CHECK-GI-NEXT: .cfi_offset w21, -24
-; CHECK-GI-NEXT: .cfi_offset w22, -32
-; CHECK-GI-NEXT: .cfi_offset w23, -40
-; CHECK-GI-NEXT: .cfi_offset w24, -48
-; CHECK-GI-NEXT: .cfi_offset w25, -56
-; CHECK-GI-NEXT: .cfi_offset w26, -64
-; CHECK-GI-NEXT: .cfi_offset w27, -80
-; CHECK-GI-NEXT: ldp q2, q1, [x0]
-; CHECK-GI-NEXT: mov v0.s[0], wzr
-; CHECK-GI-NEXT: ldrb w2, [x0, #32]
-; CHECK-GI-NEXT: umov w20, v2.b[0]
-; CHECK-GI-NEXT: umov w21, v2.b[4]
-; CHECK-GI-NEXT: umov w22, v2.b[8]
-; CHECK-GI-NEXT: umov w23, v2.b[12]
-; CHECK-GI-NEXT: umov w24, v1.b[0]
-; CHECK-GI-NEXT: umov w25, v1.b[4]
-; CHECK-GI-NEXT: umov w26, v1.b[8]
-; CHECK-GI-NEXT: umov w27, v1.b[12]
-; CHECK-GI-NEXT: umov w0, v2.b[1]
-; CHECK-GI-NEXT: umov w12, v2.b[2]
-; CHECK-GI-NEXT: umov w8, v2.b[3]
-; CHECK-GI-NEXT: umov w3, v2.b[5]
-; CHECK-GI-NEXT: umov w14, v2.b[6]
-; CHECK-GI-NEXT: umov w9, v2.b[7]
-; CHECK-GI-NEXT: umov w4, v2.b[9]
-; CHECK-GI-NEXT: umov w15, v2.b[10]
-; CHECK-GI-NEXT: umov w10, v2.b[11]
-; CHECK-GI-NEXT: umov w5, v2.b[13]
-; CHECK-GI-NEXT: umov w16, v2.b[14]
-; CHECK-GI-NEXT: umov w11, v2.b[15]
-; CHECK-GI-NEXT: umov w6, v1.b[1]
-; CHECK-GI-NEXT: umov w7, v1.b[5]
-; CHECK-GI-NEXT: umov w19, v1.b[9]
-; CHECK-GI-NEXT: mov v2.s[0], w20
-; CHECK-GI-NEXT: mov v3.s[0], w21
-; CHECK-GI-NEXT: mov v4.s[0], w22
-; CHECK-GI-NEXT: mov v5.s[0], w23
-; CHECK-GI-NEXT: mov v6.s[0], w24
-; CHECK-GI-NEXT: ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v7.s[0], w25
-; CHECK-GI-NEXT: mov v16.s[0], w26
-; CHECK-GI-NEXT: umov w20, v1.b[13]
-; CHECK-GI-NEXT: ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v17.s[0], w27
-; CHECK-GI-NEXT: mov v18.s[0], w2
-; CHECK-GI-NEXT: umov w17, v1.b[2]
-; CHECK-GI-NEXT: umov w1, v1.b[6]
-; CHECK-GI-NEXT: umov w2, v1.b[10]
-; CHECK-GI-NEXT: umov w21, v1.b[14]
-; CHECK-GI-NEXT: mov v2.s[1], w0
-; CHECK-GI-NEXT: mov v3.s[1], w3
-; CHECK-GI-NEXT: mov v4.s[1], w4
-; CHECK-GI-NEXT: mov v5.s[1], w5
-; CHECK-GI-NEXT: mov v6.s[1], w6
-; CHECK-GI-NEXT: mov v7.s[1], w7
-; CHECK-GI-NEXT: mov v16.s[1], w19
-; CHECK-GI-NEXT: mov v17.s[1], w20
-; CHECK-GI-NEXT: ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov v18.s[1], wzr
+; CHECK-GI-NEXT: ldp q7, q19, [x0]
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: ldrb w10, [x0, #32]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s3, wzr
+; CHECK-GI-NEXT: fmov s2, wzr
+; CHECK-GI-NEXT: fmov s5, wzr
+; CHECK-GI-NEXT: fmov s4, wzr
+; CHECK-GI-NEXT: umov w15, v7.b[8]
+; CHECK-GI-NEXT: umov w2, v7.b[12]
+; CHECK-GI-NEXT: umov w16, v7.b[9]
+; CHECK-GI-NEXT: umov w4, v7.b[13]
+; CHECK-GI-NEXT: umov w8, v7.b[0]
+; CHECK-GI-NEXT: umov w12, v7.b[4]
+; CHECK-GI-NEXT: umov w9, v7.b[1]
+; CHECK-GI-NEXT: umov w14, v7.b[5]
+; CHECK-GI-NEXT: umov w3, v7.b[10]
+; CHECK-GI-NEXT: umov w5, v7.b[14]
+; CHECK-GI-NEXT: umov w11, v7.b[2]
+; CHECK-GI-NEXT: umov w17, v7.b[3]
+; CHECK-GI-NEXT: fmov s17, w15
+; CHECK-GI-NEXT: fmov s18, w2
+; CHECK-GI-NEXT: umov w13, v7.b[6]
+; CHECK-GI-NEXT: umov w18, v7.b[7]
+; CHECK-GI-NEXT: umov w0, v7.b[11]
+; CHECK-GI-NEXT: umov w1, v7.b[15]
+; CHECK-GI-NEXT: fmov s7, w8
+; CHECK-GI-NEXT: fmov s16, w12
+; CHECK-GI-NEXT: umov w2, v19.b[0]
+; CHECK-GI-NEXT: mov v17.s[1], w16
+; CHECK-GI-NEXT: mov v18.s[1], w4
+; CHECK-GI-NEXT: umov w4, v19.b[4]
+; CHECK-GI-NEXT: umov w6, v19.b[1]
+; CHECK-GI-NEXT: umov w7, v19.b[5]
+; CHECK-GI-NEXT: umov w19, v19.b[9]
+; CHECK-GI-NEXT: mov v7.s[1], w9
+; CHECK-GI-NEXT: mov v16.s[1], w14
+; CHECK-GI-NEXT: umov w20, v19.b[13]
+; CHECK-GI-NEXT: fmov s6, wzr
+; CHECK-GI-NEXT: umov w12, v19.b[2]
+; CHECK-GI-NEXT: umov w8, v19.b[3]
+; CHECK-GI-NEXT: mov v17.s[2], w3
+; CHECK-GI-NEXT: umov w3, v19.b[8]
+; CHECK-GI-NEXT: mov v18.s[2], w5
+; CHECK-GI-NEXT: umov w5, v19.b[12]
+; CHECK-GI-NEXT: umov w14, v19.b[6]
+; CHECK-GI-NEXT: umov w9, v19.b[7]
+; CHECK-GI-NEXT: mov v7.s[2], w11
+; CHECK-GI-NEXT: mov v16.s[2], w13
+; CHECK-GI-NEXT: umov w15, v19.b[10]
+; CHECK-GI-NEXT: umov w11, v19.b[11]
+; CHECK-GI-NEXT: umov w16, v19.b[14]
+; CHECK-GI-NEXT: umov w13, v19.b[15]
+; CHECK-GI-NEXT: fmov s19, w2
+; CHECK-GI-NEXT: fmov s20, w4
+; CHECK-GI-NEXT: fmov s21, w3
+; CHECK-GI-NEXT: fmov s22, w5
+; CHECK-GI-NEXT: fmov s23, w10
+; CHECK-GI-NEXT: mov v1.s[1], wzr
; CHECK-GI-NEXT: mov v0.s[1], wzr
-; CHECK-GI-NEXT: umov w13, v1.b[3]
-; CHECK-GI-NEXT: umov w18, v1.b[7]
-; CHECK-GI-NEXT: umov w0, v1.b[11]
-; CHECK-GI-NEXT: umov w3, v1.b[15]
-; CHECK-GI-NEXT: mov v2.s[2], w12
-; CHECK-GI-NEXT: mov v3.s[2], w14
-; CHECK-GI-NEXT: mov v4.s[2], w15
-; CHECK-GI-NEXT: mov v5.s[2], w16
-; CHECK-GI-NEXT: mov v6.s[2], w17
-; CHECK-GI-NEXT: mov v7.s[2], w1
-; CHECK-GI-NEXT: mov v16.s[2], w2
-; CHECK-GI-NEXT: mov v17.s[2], w21
-; CHECK-GI-NEXT: mov v18.s[2], wzr
-; CHECK-GI-NEXT: ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v3.s[1], wzr
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v19.s[1], w6
+; CHECK-GI-NEXT: mov v20.s[1], w7
+; CHECK-GI-NEXT: mov v21.s[1], w19
+; CHECK-GI-NEXT: mov v22.s[1], w20
+; CHECK-GI-NEXT: mov v23.s[1], wzr
+; CHECK-GI-NEXT: mov v5.s[1], wzr
+; CHECK-GI-NEXT: mov v4.s[1], wzr
+; CHECK-GI-NEXT: mov v6.s[1], wzr
+; CHECK-GI-NEXT: mov v1.s[2], wzr
; CHECK-GI-NEXT: mov v0.s[2], wzr
-; CHECK-GI-NEXT: mov v2.s[3], w8
-; CHECK-GI-NEXT: mov v3.s[3], w9
-; CHECK-GI-NEXT: mov v4.s[3], w10
-; CHECK-GI-NEXT: mov v5.s[3], w11
-; CHECK-GI-NEXT: mov v6.s[3], w13
-; CHECK-GI-NEXT: mov v7.s[3], w18
-; CHECK-GI-NEXT: mov v16.s[3], w0
-; CHECK-GI-NEXT: mov v17.s[3], w3
-; CHECK-GI-NEXT: mov v18.s[3], wzr
+; CHECK-GI-NEXT: mov v3.s[2], wzr
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v19.s[2], w12
+; CHECK-GI-NEXT: mov v20.s[2], w14
+; CHECK-GI-NEXT: mov v21.s[2], w15
+; CHECK-GI-NEXT: mov v22.s[2], w16
+; CHECK-GI-NEXT: mov v23.s[2], wzr
+; CHECK-GI-NEXT: mov v5.s[2], wzr
+; CHECK-GI-NEXT: mov v4.s[2], wzr
+; CHECK-GI-NEXT: mov v6.s[2], wzr
+; CHECK-GI-NEXT: mov v7.s[3], w17
+; CHECK-GI-NEXT: mov v16.s[3], w18
+; CHECK-GI-NEXT: mov v17.s[3], w0
+; CHECK-GI-NEXT: mov v18.s[3], w1
+; CHECK-GI-NEXT: mov v19.s[3], w8
+; CHECK-GI-NEXT: mov v20.s[3], w9
+; CHECK-GI-NEXT: mov v21.s[3], w11
+; CHECK-GI-NEXT: mov v22.s[3], w13
+; CHECK-GI-NEXT: mov v23.s[3], wzr
+; CHECK-GI-NEXT: mov v1.s[3], wzr
; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: mov v3.s[3], wzr
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mov v5.s[3], wzr
+; CHECK-GI-NEXT: mov v4.s[3], wzr
+; CHECK-GI-NEXT: mov v6.s[3], wzr
+; CHECK-GI-NEXT: add v7.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT: add v16.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: add v17.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT: add v18.4s, v21.4s, v22.4s
+; CHECK-GI-NEXT: add v1.4s, v23.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT: add v3.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT: add v4.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT: add v5.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT: add v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT: add v5.4s, v18.4s, v0.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: add v3.4s, v5.4s, v0.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
; CHECK-GI-NEXT: addv s0, v0.4s
; CHECK-GI-NEXT: fmov w0, s0
-; CHECK-GI-NEXT: ldr x27, [sp], #80 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldp x20, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%0 = load <33 x i8>, ptr %a1
@@ -4266,169 +4316,199 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_sdot_v33i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldp q19, q4, [x1]
-; CHECK-GI-NEXT: mov v25.s[0], wzr
-; CHECK-GI-NEXT: ldp q7, q5, [x0]
-; CHECK-GI-NEXT: smov w8, v19.b[0]
-; CHECK-GI-NEXT: smov w9, v19.b[4]
-; CHECK-GI-NEXT: smov w10, v19.b[8]
-; CHECK-GI-NEXT: smov w11, v19.b[1]
-; CHECK-GI-NEXT: smov w12, v19.b[6]
-; CHECK-GI-NEXT: smov w13, v19.b[12]
-; CHECK-GI-NEXT: smov w14, v4.b[0]
-; CHECK-GI-NEXT: smov w15, v4.b[4]
-; CHECK-GI-NEXT: smov w16, v4.b[12]
-; CHECK-GI-NEXT: mov v25.s[1], wzr
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: smov w8, v19.b[5]
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: smov w9, v19.b[9]
-; CHECK-GI-NEXT: mov v1.s[0], w10
-; CHECK-GI-NEXT: smov w10, v19.b[2]
-; CHECK-GI-NEXT: mov v6.s[0], w13
-; CHECK-GI-NEXT: smov w13, v19.b[3]
-; CHECK-GI-NEXT: mov v3.s[0], w14
-; CHECK-GI-NEXT: smov w14, v19.b[13]
-; CHECK-GI-NEXT: mov v16.s[0], w15
-; CHECK-GI-NEXT: smov w15, v4.b[8]
-; CHECK-GI-NEXT: mov v0.s[1], w11
-; CHECK-GI-NEXT: smov w11, v19.b[10]
-; CHECK-GI-NEXT: mov v2.s[1], w8
-; CHECK-GI-NEXT: ldrsb w8, [x0, #32]
-; CHECK-GI-NEXT: mov v1.s[1], w9
-; CHECK-GI-NEXT: ldrsb w9, [x1, #32]
-; CHECK-GI-NEXT: mov v17.s[0], w16
-; CHECK-GI-NEXT: smov w16, v19.b[14]
-; CHECK-GI-NEXT: mov v25.s[2], wzr
-; CHECK-GI-NEXT: mul w8, w9, w8
-; CHECK-GI-NEXT: mov v6.s[1], w14
-; CHECK-GI-NEXT: smov w14, v4.b[1]
-; CHECK-GI-NEXT: mov v0.s[2], w10
-; CHECK-GI-NEXT: smov w10, v19.b[7]
-; CHECK-GI-NEXT: mov v2.s[2], w12
-; CHECK-GI-NEXT: smov w12, v19.b[11]
-; CHECK-GI-NEXT: mov v1.s[2], w11
-; CHECK-GI-NEXT: smov w11, v4.b[5]
-; CHECK-GI-NEXT: mov v18.s[0], w15
-; CHECK-GI-NEXT: smov w15, v19.b[15]
-; CHECK-GI-NEXT: smov w9, v5.b[2]
-; CHECK-GI-NEXT: mov v6.s[2], w16
-; CHECK-GI-NEXT: smov w16, v7.b[0]
-; CHECK-GI-NEXT: mov v3.s[1], w14
-; CHECK-GI-NEXT: mov v0.s[3], w13
-; CHECK-GI-NEXT: smov w13, v7.b[4]
-; CHECK-GI-NEXT: mov v2.s[3], w10
-; CHECK-GI-NEXT: smov w10, v4.b[6]
-; CHECK-GI-NEXT: mov v1.s[3], w12
-; CHECK-GI-NEXT: smov w12, v4.b[13]
-; CHECK-GI-NEXT: mov v16.s[1], w11
-; CHECK-GI-NEXT: smov w11, v4.b[9]
-; CHECK-GI-NEXT: smov w14, v7.b[5]
-; CHECK-GI-NEXT: mov v19.s[0], w16
-; CHECK-GI-NEXT: smov w16, v7.b[1]
-; CHECK-GI-NEXT: mov v6.s[3], w15
-; CHECK-GI-NEXT: mov v20.s[0], w13
-; CHECK-GI-NEXT: smov w13, v4.b[2]
-; CHECK-GI-NEXT: smov w15, v7.b[6]
-; CHECK-GI-NEXT: mov v17.s[1], w12
-; CHECK-GI-NEXT: smov w12, v4.b[14]
-; CHECK-GI-NEXT: mov v27.s[0], w8
-; CHECK-GI-NEXT: mov v16.s[2], w10
-; CHECK-GI-NEXT: smov w10, v4.b[7]
-; CHECK-GI-NEXT: mov v18.s[1], w11
-; CHECK-GI-NEXT: smov w11, v4.b[10]
-; CHECK-GI-NEXT: mov v19.s[1], w16
-; CHECK-GI-NEXT: smov w16, v5.b[4]
+; CHECK-GI-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: .cfi_offset b8, -16
+; CHECK-GI-NEXT: ldp q21, q25, [x1]
+; CHECK-GI-NEXT: fmov s5, wzr
+; CHECK-GI-NEXT: ldp q26, q22, [x0]
+; CHECK-GI-NEXT: fmov s6, wzr
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s3, wzr
+; CHECK-GI-NEXT: smov w8, v21.b[0]
+; CHECK-GI-NEXT: smov w9, v21.b[4]
+; CHECK-GI-NEXT: smov w10, v21.b[1]
+; CHECK-GI-NEXT: smov w13, v21.b[8]
+; CHECK-GI-NEXT: smov w11, v21.b[5]
+; CHECK-GI-NEXT: smov w14, v21.b[9]
+; CHECK-GI-NEXT: smov w15, v25.b[0]
+; CHECK-GI-NEXT: smov w12, v21.b[2]
+; CHECK-GI-NEXT: fmov s2, wzr
+; CHECK-GI-NEXT: fmov s4, wzr
+; CHECK-GI-NEXT: mov v5.s[1], wzr
+; CHECK-GI-NEXT: mov v6.s[1], wzr
+; CHECK-GI-NEXT: fmov s7, w8
+; CHECK-GI-NEXT: fmov s17, w9
+; CHECK-GI-NEXT: smov w8, v21.b[6]
+; CHECK-GI-NEXT: fmov s16, w13
+; CHECK-GI-NEXT: smov w9, v21.b[3]
+; CHECK-GI-NEXT: smov w13, v21.b[7]
+; CHECK-GI-NEXT: fmov s18, w15
+; CHECK-GI-NEXT: smov w15, v25.b[4]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v7.s[1], w10
+; CHECK-GI-NEXT: smov w10, v21.b[12]
+; CHECK-GI-NEXT: mov v17.s[1], w11
+; CHECK-GI-NEXT: smov w11, v21.b[13]
+; CHECK-GI-NEXT: mov v16.s[1], w14
+; CHECK-GI-NEXT: smov w14, v25.b[1]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], wzr
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: fmov s20, w15
+; CHECK-GI-NEXT: smov w15, v25.b[13]
+; CHECK-GI-NEXT: mov v4.s[1], wzr
+; CHECK-GI-NEXT: fmov s19, w10
+; CHECK-GI-NEXT: mov v7.s[2], w12
+; CHECK-GI-NEXT: smov w12, v21.b[10]
+; CHECK-GI-NEXT: mov v18.s[1], w14
+; CHECK-GI-NEXT: smov w14, v25.b[5]
+; CHECK-GI-NEXT: mov v17.s[2], w8
+; CHECK-GI-NEXT: smov w8, v21.b[11]
+; CHECK-GI-NEXT: smov w10, v21.b[14]
+; CHECK-GI-NEXT: mov v5.s[2], wzr
+; CHECK-GI-NEXT: mov v19.s[1], w11
+; CHECK-GI-NEXT: smov w11, v25.b[2]
+; CHECK-GI-NEXT: mov v6.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[2], w12
+; CHECK-GI-NEXT: smov w12, v25.b[8]
+; CHECK-GI-NEXT: mov v7.s[3], w9
; CHECK-GI-NEXT: mov v20.s[1], w14
-; CHECK-GI-NEXT: smov w14, v4.b[15]
-; CHECK-GI-NEXT: mov v3.s[2], w13
-; CHECK-GI-NEXT: mov v17.s[2], w12
-; CHECK-GI-NEXT: smov w12, v7.b[12]
-; CHECK-GI-NEXT: smov w13, v7.b[7]
-; CHECK-GI-NEXT: mov v16.s[3], w10
-; CHECK-GI-NEXT: smov w10, v7.b[8]
-; CHECK-GI-NEXT: smov w8, v7.b[3]
+; CHECK-GI-NEXT: smov w14, v21.b[15]
+; CHECK-GI-NEXT: smov w9, v25.b[9]
+; CHECK-GI-NEXT: mov v17.s[3], w13
+; CHECK-GI-NEXT: smov w13, v25.b[12]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
; CHECK-GI-NEXT: mov v18.s[2], w11
-; CHECK-GI-NEXT: smov w11, v7.b[2]
-; CHECK-GI-NEXT: mov v23.s[0], w16
-; CHECK-GI-NEXT: mov v20.s[2], w15
-; CHECK-GI-NEXT: smov w15, v5.b[12]
-; CHECK-GI-NEXT: smov w16, v7.b[14]
-; CHECK-GI-NEXT: mov v17.s[3], w14
-; CHECK-GI-NEXT: smov w14, v7.b[13]
-; CHECK-GI-NEXT: mov v22.s[0], w12
-; CHECK-GI-NEXT: smov w12, v7.b[9]
-; CHECK-GI-NEXT: mov v21.s[0], w10
-; CHECK-GI-NEXT: smov w10, v4.b[3]
-; CHECK-GI-NEXT: mov v19.s[2], w11
-; CHECK-GI-NEXT: smov w11, v5.b[0]
-; CHECK-GI-NEXT: mov v27.s[1], wzr
-; CHECK-GI-NEXT: mov v20.s[3], w13
-; CHECK-GI-NEXT: smov w13, v5.b[5]
-; CHECK-GI-NEXT: mov v24.s[0], w15
-; CHECK-GI-NEXT: mov v22.s[1], w14
-; CHECK-GI-NEXT: smov w14, v5.b[8]
-; CHECK-GI-NEXT: smov w15, v4.b[11]
-; CHECK-GI-NEXT: mov v21.s[1], w12
-; CHECK-GI-NEXT: smov w12, v5.b[13]
-; CHECK-GI-NEXT: mov v25.s[3], wzr
-; CHECK-GI-NEXT: mov v4.s[0], w11
-; CHECK-GI-NEXT: smov w11, v5.b[1]
-; CHECK-GI-NEXT: mov v3.s[3], w10
-; CHECK-GI-NEXT: mov v23.s[1], w13
-; CHECK-GI-NEXT: smov w13, v5.b[6]
-; CHECK-GI-NEXT: mov v19.s[3], w8
-; CHECK-GI-NEXT: mov v22.s[2], w16
-; CHECK-GI-NEXT: smov w16, v5.b[9]
-; CHECK-GI-NEXT: mov v26.s[0], w14
+; CHECK-GI-NEXT: smov w11, v26.b[0]
+; CHECK-GI-NEXT: mov v19.s[2], w10
+; CHECK-GI-NEXT: fmov s21, w12
+; CHECK-GI-NEXT: smov w12, v26.b[1]
+; CHECK-GI-NEXT: mov v16.s[3], w8
+; CHECK-GI-NEXT: smov w8, v26.b[5]
+; CHECK-GI-NEXT: smov w10, v25.b[6]
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: fmov s23, w13
+; CHECK-GI-NEXT: smov w13, v25.b[3]
+; CHECK-GI-NEXT: mov v3.s[2], wzr
+; CHECK-GI-NEXT: fmov s24, w11
+; CHECK-GI-NEXT: mov v21.s[1], w9
+; CHECK-GI-NEXT: smov w9, v25.b[10]
+; CHECK-GI-NEXT: smov w11, v26.b[2]
+; CHECK-GI-NEXT: mov v19.s[3], w14
+; CHECK-GI-NEXT: smov w14, v26.b[13]
+; CHECK-GI-NEXT: mov v23.s[1], w15
+; CHECK-GI-NEXT: smov w15, v25.b[14]
+; CHECK-GI-NEXT: mov v20.s[2], w10
; CHECK-GI-NEXT: mov v24.s[1], w12
-; CHECK-GI-NEXT: smov w12, v5.b[14]
-; CHECK-GI-NEXT: smov w14, v7.b[10]
-; CHECK-GI-NEXT: mov v4.s[1], w11
-; CHECK-GI-NEXT: smov w11, v7.b[15]
-; CHECK-GI-NEXT: mov v18.s[3], w15
-; CHECK-GI-NEXT: mov v23.s[2], w13
-; CHECK-GI-NEXT: smov w13, v5.b[7]
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v20.4s
-; CHECK-GI-NEXT: mov v26.s[1], w16
-; CHECK-GI-NEXT: smov w16, v5.b[10]
-; CHECK-GI-NEXT: mov v27.s[2], wzr
-; CHECK-GI-NEXT: mov v24.s[2], w12
-; CHECK-GI-NEXT: smov w12, v5.b[15]
-; CHECK-GI-NEXT: mov v21.s[2], w14
-; CHECK-GI-NEXT: smov w14, v7.b[11]
-; CHECK-GI-NEXT: mov v4.s[2], w9
-; CHECK-GI-NEXT: smov w9, v5.b[3]
-; CHECK-GI-NEXT: mov v22.s[3], w11
-; CHECK-GI-NEXT: smov w11, v5.b[11]
-; CHECK-GI-NEXT: mov v23.s[3], w13
-; CHECK-GI-NEXT: mov v26.s[2], w16
-; CHECK-GI-NEXT: mla v2.4s, v0.4s, v19.4s
-; CHECK-GI-NEXT: mov v27.s[3], wzr
-; CHECK-GI-NEXT: mov v24.s[3], w12
-; CHECK-GI-NEXT: mov v21.s[3], w14
-; CHECK-GI-NEXT: mov v4.s[3], w9
-; CHECK-GI-NEXT: mul v5.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT: mul v6.4s, v16.4s, v23.4s
-; CHECK-GI-NEXT: add v16.4s, v25.4s, v25.4s
-; CHECK-GI-NEXT: mov v26.s[3], w11
-; CHECK-GI-NEXT: mul v7.4s, v17.4s, v24.4s
-; CHECK-GI-NEXT: add v0.4s, v25.4s, v16.4s
-; CHECK-GI-NEXT: mla v5.4s, v1.4s, v21.4s
-; CHECK-GI-NEXT: mla v6.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: add v3.4s, v16.4s, v16.4s
-; CHECK-GI-NEXT: mla v7.4s, v18.4s, v26.4s
-; CHECK-GI-NEXT: add v0.4s, v27.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: add w0, w8, w2
-; CHECK-GI-NEXT: ret
-entry:
+; CHECK-GI-NEXT: smov w12, v26.b[4]
+; CHECK-GI-NEXT: smov w10, v25.b[7]
+; CHECK-GI-NEXT: mov v21.s[2], w9
+; CHECK-GI-NEXT: smov w9, v25.b[11]
+; CHECK-GI-NEXT: mov v18.s[3], w13
+; CHECK-GI-NEXT: smov w13, v26.b[9]
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v4.s[2], wzr
+; CHECK-GI-NEXT: mov v23.s[2], w15
+; CHECK-GI-NEXT: smov w15, v25.b[15]
+; CHECK-GI-NEXT: mov v5.s[3], wzr
+; CHECK-GI-NEXT: fmov s27, w12
+; CHECK-GI-NEXT: mov v24.s[2], w11
+; CHECK-GI-NEXT: smov w11, v26.b[6]
+; CHECK-GI-NEXT: smov w12, v26.b[8]
+; CHECK-GI-NEXT: mov v21.s[3], w9
+; CHECK-GI-NEXT: smov w9, v26.b[12]
+; CHECK-GI-NEXT: mov v20.s[3], w10
+; CHECK-GI-NEXT: smov w10, v26.b[3]
+; CHECK-GI-NEXT: mov v6.s[3], wzr
+; CHECK-GI-NEXT: mov v27.s[1], w8
+; CHECK-GI-NEXT: mov v23.s[3], w15
+; CHECK-GI-NEXT: smov w15, v22.b[0]
+; CHECK-GI-NEXT: smov w8, v26.b[7]
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: fmov s25, w12
+; CHECK-GI-NEXT: fmov s29, w9
+; CHECK-GI-NEXT: smov w9, v22.b[5]
+; CHECK-GI-NEXT: mov v24.s[3], w10
+; CHECK-GI-NEXT: smov w10, v22.b[1]
+; CHECK-GI-NEXT: smov w12, v26.b[10]
+; CHECK-GI-NEXT: mov v27.s[2], w11
+; CHECK-GI-NEXT: smov w11, v22.b[4]
+; CHECK-GI-NEXT: fmov s28, w15
+; CHECK-GI-NEXT: mov v25.s[1], w13
+; CHECK-GI-NEXT: smov w13, v26.b[14]
+; CHECK-GI-NEXT: mov v29.s[1], w14
+; CHECK-GI-NEXT: smov w15, v22.b[12]
+; CHECK-GI-NEXT: smov w14, v22.b[2]
+; CHECK-GI-NEXT: mov v3.s[3], wzr
+; CHECK-GI-NEXT: mov v28.s[1], w10
+; CHECK-GI-NEXT: smov w10, v22.b[13]
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: fmov s30, w11
+; CHECK-GI-NEXT: smov w11, v22.b[6]
+; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: mov v25.s[2], w12
+; CHECK-GI-NEXT: mov v29.s[2], w13
+; CHECK-GI-NEXT: smov w13, v26.b[11]
+; CHECK-GI-NEXT: fmov s31, w15
+; CHECK-GI-NEXT: smov w15, v26.b[15]
+; CHECK-GI-NEXT: smov w12, v22.b[9]
+; CHECK-GI-NEXT: mov v30.s[1], w9
+; CHECK-GI-NEXT: smov w9, v22.b[8]
+; CHECK-GI-NEXT: mov v28.s[2], w14
+; CHECK-GI-NEXT: ldrsb w14, [x1, #32]
+; CHECK-GI-NEXT: smov w8, v22.b[15]
+; CHECK-GI-NEXT: mul v17.4s, v17.4s, v27.4s
+; CHECK-GI-NEXT: mov v31.s[1], w10
+; CHECK-GI-NEXT: smov w10, v22.b[14]
+; CHECK-GI-NEXT: mov v25.s[3], w13
+; CHECK-GI-NEXT: ldrsb w13, [x0, #32]
+; CHECK-GI-NEXT: mov v29.s[3], w15
+; CHECK-GI-NEXT: mov v4.s[3], wzr
+; CHECK-GI-NEXT: mov v30.s[2], w11
+; CHECK-GI-NEXT: fmov s26, w9
+; CHECK-GI-NEXT: smov w9, v22.b[7]
+; CHECK-GI-NEXT: smov w11, v22.b[3]
+; CHECK-GI-NEXT: add v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: mla v17.4s, v7.4s, v24.4s
+; CHECK-GI-NEXT: mov v31.s[2], w10
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: mov v26.s[1], w12
+; CHECK-GI-NEXT: smov w12, v22.b[10]
+; CHECK-GI-NEXT: mul v19.4s, v19.4s, v29.4s
+; CHECK-GI-NEXT: mov v30.s[3], w9
+; CHECK-GI-NEXT: mul w9, w14, w13
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: mov v28.s[3], w11
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT: mov v31.s[3], w8
+; CHECK-GI-NEXT: smov w8, v22.b[11]
+; CHECK-GI-NEXT: fmov s8, w9
+; CHECK-GI-NEXT: mov v26.s[2], w12
+; CHECK-GI-NEXT: mla v19.4s, v16.4s, v25.4s
+; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: mov v8.s[1], wzr
+; CHECK-GI-NEXT: mul v22.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT: mov v26.s[3], w8
+; CHECK-GI-NEXT: add v3.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT: mla v20.4s, v18.4s, v28.4s
+; CHECK-GI-NEXT: mov v8.s[2], wzr
+; CHECK-GI-NEXT: mla v22.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT: mov v8.s[3], wzr
+; CHECK-GI-NEXT: add v4.4s, v20.4s, v22.4s
+; CHECK-GI-NEXT: add v0.4s, v8.4s, v0.4s
+; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ret
+entry:
%0 = load <33 x i8>, ptr %a
%1 = sext <33 x i8> %0 to <33 x i32>
%2 = load <33 x i8>, ptr %b
@@ -4722,12 +4802,13 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
;
; CHECK-GI-LABEL: test_sdot_v33i8_double:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT: str x29, [sp, #64] // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT: sub sp, sp, #96
+; CHECK-GI-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT: str x29, [sp, #80] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 96
; CHECK-GI-NEXT: .cfi_offset w29, -16
; CHECK-GI-NEXT: .cfi_offset b8, -24
; CHECK-GI-NEXT: .cfi_offset b9, -32
@@ -4737,444 +4818,508 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33
; CHECK-GI-NEXT: .cfi_offset b13, -64
; CHECK-GI-NEXT: .cfi_offset b14, -72
; CHECK-GI-NEXT: .cfi_offset b15, -80
-; CHECK-GI-NEXT: ldr w8, [sp, #80]
-; CHECK-GI-NEXT: sxtb w9, w0
-; CHECK-GI-NEXT: ldr w10, [sp, #112]
+; CHECK-GI-NEXT: sxtb w8, w0
+; CHECK-GI-NEXT: sxtb w9, w1
+; CHECK-GI-NEXT: sxtb w10, w2
; CHECK-GI-NEXT: sxtb w11, w4
+; CHECK-GI-NEXT: sxtb w12, w5
; CHECK-GI-NEXT: sxtb w13, w7
-; CHECK-GI-NEXT: sxtb w12, w3
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v0.s[0], w9
-; CHECK-GI-NEXT: sxtb w9, w10
-; CHECK-GI-NEXT: mov v3.s[0], w11
-; CHECK-GI-NEXT: sxtb w10, w1
-; CHECK-GI-NEXT: sxtb w11, w5
-; CHECK-GI-NEXT: mov v1.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: mov v5.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #120]
-; CHECK-GI-NEXT: ldr w14, [sp, #168]
-; CHECK-GI-NEXT: ldr x29, [sp, #64] // 8-byte Folded Reload
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v0.s[1], w10
-; CHECK-GI-NEXT: sxtb w10, w2
-; CHECK-GI-NEXT: mov v3.s[1], w11
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: sxtb w11, w6
-; CHECK-GI-NEXT: mov v1.s[1], w8
+; CHECK-GI-NEXT: fmov s28, w8
; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: mov v5.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #128]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s25, w11
+; CHECK-GI-NEXT: sxtb w11, w6
+; CHECK-GI-NEXT: ldr w14, [sp, #528]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v0.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #104]
-; CHECK-GI-NEXT: mov v3.s[2], w11
+; CHECK-GI-NEXT: fmov s18, wzr
+; CHECK-GI-NEXT: fmov s20, wzr
+; CHECK-GI-NEXT: mov v28.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #104]
+; CHECK-GI-NEXT: str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT: fmov s24, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #112]
+; CHECK-GI-NEXT: mov v25.s[1], w12
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w11, [sp, #136]
-; CHECK-GI-NEXT: mov v1.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #144]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v5.s[2], w9
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #152]
+; CHECK-GI-NEXT: ldr w12, [sp, #136]
+; CHECK-GI-NEXT: mov v18.s[1], wzr
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v0.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #160]
-; CHECK-GI-NEXT: mov v3.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #176]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v1.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #208]
-; CHECK-GI-NEXT: mov v2.s[0], w8
-; CHECK-GI-NEXT: mov v5.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #240]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: sxtb w8, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #184]
+; CHECK-GI-NEXT: mov v20.s[1], wzr
+; CHECK-GI-NEXT: fmov s19, wzr
+; CHECK-GI-NEXT: mov v28.s[2], w10
+; CHECK-GI-NEXT: sxtb w10, w3
+; CHECK-GI-NEXT: mov v24.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #128]
+; CHECK-GI-NEXT: mov v25.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #168]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v6.s[0], w13
+; CHECK-GI-NEXT: mov v18.s[2], wzr
+; CHECK-GI-NEXT: fmov s21, wzr
+; CHECK-GI-NEXT: sxtb w9, w9
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w13, [sp, #264]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #216]
+; CHECK-GI-NEXT: mov v20.s[2], wzr
+; CHECK-GI-NEXT: mov v28.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #160]
+; CHECK-GI-NEXT: mov v24.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #120]
+; CHECK-GI-NEXT: fmov s30, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #144]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v7.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #248]
-; CHECK-GI-NEXT: mov v2.s[1], w9
+; CHECK-GI-NEXT: mov v25.s[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #200]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w9, [sp, #192]
-; CHECK-GI-NEXT: mov v6.s[1], w10
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w10, [sp, #200]
-; CHECK-GI-NEXT: mov v4.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #224]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v7.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #256]
-; CHECK-GI-NEXT: mov v2.s[2], w12
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w12, [sp, #232]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v6.s[2], w9
-; CHECK-GI-NEXT: sxtb w9, w11
-; CHECK-GI-NEXT: sxtb w11, w14
-; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: ldr w14, [sp, #280]
-; CHECK-GI-NEXT: ldr w8, [sp, #272]
-; CHECK-GI-NEXT: mov v7.s[2], w9
-; CHECK-GI-NEXT: mov v2.s[3], w11
-; CHECK-GI-NEXT: sxtb w11, w12
-; CHECK-GI-NEXT: sxtb w12, w13
-; CHECK-GI-NEXT: sxtb w13, w14
-; CHECK-GI-NEXT: ldr w9, [sp, #288]
-; CHECK-GI-NEXT: mov v6.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #312]
-; CHECK-GI-NEXT: ldr w14, [sp, #544]
-; CHECK-GI-NEXT: mov v4.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #344]
-; CHECK-GI-NEXT: mov v16.s[0], w13
-; CHECK-GI-NEXT: mov v7.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #376]
+; CHECK-GI-NEXT: mov v19.s[1], wzr
+; CHECK-GI-NEXT: fmov s22, w10
+; CHECK-GI-NEXT: mov v30.s[1], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #176]
+; CHECK-GI-NEXT: mov v24.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #224]
+; CHECK-GI-NEXT: ldr w12, [sp, #152]
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v21.s[1], wzr
+; CHECK-GI-NEXT: mov v22.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #192]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v30.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #232]
+; CHECK-GI-NEXT: sxtb w12, w12
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: fmov s23, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #240]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w13, [sp, #296]
-; CHECK-GI-NEXT: mov v19.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #320]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #352]
-; CHECK-GI-NEXT: mov v16.s[1], w9
-; CHECK-GI-NEXT: mov v21.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #384]
+; CHECK-GI-NEXT: mov v18.s[3], wzr
+; CHECK-GI-NEXT: mov v20.s[3], wzr
+; CHECK-GI-NEXT: mov v22.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #184]
+; CHECK-GI-NEXT: fmov s26, w11
+; CHECK-GI-NEXT: mov v23.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: ldr w11, [sp, #208]
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v30.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #264]
+; CHECK-GI-NEXT: mov v26.s[1], w13
+; CHECK-GI-NEXT: sxtb w9, w9
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w9, [sp, #304]
-; CHECK-GI-NEXT: mov v19.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #328]
+; CHECK-GI-NEXT: mov v22.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #296]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: fmov s29, w9
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #360]
-; CHECK-GI-NEXT: mov v16.s[2], w13
-; CHECK-GI-NEXT: mov v21.s[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #392]
-; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: ldr w13, [sp, #216]
+; CHECK-GI-NEXT: sxtb w9, w10
+; CHECK-GI-NEXT: mov v23.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #248]
+; CHECK-GI-NEXT: mov v26.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #304]
+; CHECK-GI-NEXT: ldr w10, [sp, #272]
+; CHECK-GI-NEXT: fmov s31, w9
+; CHECK-GI-NEXT: mov v29.s[1], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #312]
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: ldr w12, [sp, #280]
+; CHECK-GI-NEXT: fmov s16, wzr
+; CHECK-GI-NEXT: mov v31.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #328]
+; CHECK-GI-NEXT: mov v23.s[3], w8
+; CHECK-GI-NEXT: sxtb w8, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #360]
+; CHECK-GI-NEXT: mov v29.s[2], w10
+; CHECK-GI-NEXT: sxtb w10, w11
+; CHECK-GI-NEXT: mov v26.s[3], w13
; CHECK-GI-NEXT: ldr w13, [sp, #336]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v19.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #368]
+; CHECK-GI-NEXT: ldr w11, [sp, #368]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[2], w11
+; CHECK-GI-NEXT: mov v31.s[2], w8
+; CHECK-GI-NEXT: fmov s0, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #320]
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: ldr w11, [sp, #400]
-; CHECK-GI-NEXT: mov v21.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #408]
+; CHECK-GI-NEXT: fmov s12, w9
+; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v29.s[3], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #376]
+; CHECK-GI-NEXT: mov v0.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #344]
+; CHECK-GI-NEXT: ldr w8, [sp, #288]
+; CHECK-GI-NEXT: mov v12.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #392]
+; CHECK-GI-NEXT: mov v31.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #424]
+; CHECK-GI-NEXT: sxtb w12, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #400]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v16.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #416]
-; CHECK-GI-NEXT: mov v19.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #440]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #472]
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v0.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #432]
+; CHECK-GI-NEXT: fmov s13, w11
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v18.s[0], w12
-; CHECK-GI-NEXT: mov v21.s[3], w11
+; CHECK-GI-NEXT: mov v12.s[2], w9
+; CHECK-GI-NEXT: fmov s8, w10
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w10, [sp, #440]
+; CHECK-GI-NEXT: ldr w11, [sp, #384]
+; CHECK-GI-NEXT: ldr w9, [sp, #352]
+; CHECK-GI-NEXT: fmov s17, wzr
+; CHECK-GI-NEXT: mov v13.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #408]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w11, [sp, #504]
-; CHECK-GI-NEXT: ldr w12, [sp, #424]
-; CHECK-GI-NEXT: mov v22.s[0], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #448]
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v20.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #480]
+; CHECK-GI-NEXT: mov v8.s[1], w12
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w12, [sp, #456]
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v18.s[1], w9
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: fmov s3, wzr
+; CHECK-GI-NEXT: mov v12.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #488]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v23.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #512]
-; CHECK-GI-NEXT: mov v22.s[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #456]
-; CHECK-GI-NEXT: ldr w9, [sp, #432]
-; CHECK-GI-NEXT: mov v20.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #488]
+; CHECK-GI-NEXT: mov v13.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #496]
+; CHECK-GI-NEXT: mov v0.s[3], w9
+; CHECK-GI-NEXT: mov v8.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #416]
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w9, [sp, #464]
+; CHECK-GI-NEXT: fmov s14, w12
; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v18.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #464]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v23.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #520]
-; CHECK-GI-NEXT: mov v22.s[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #496]
+; CHECK-GI-NEXT: fmov s9, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #504]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v20.s[2], w10
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v18.s[3], w9
-; CHECK-GI-NEXT: sxtb w9, w13
-; CHECK-GI-NEXT: ldr w10, [sp, #528]
-; CHECK-GI-NEXT: mov v23.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #536]
-; CHECK-GI-NEXT: sxtb w13, w14
-; CHECK-GI-NEXT: mov v22.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #576]
+; CHECK-GI-NEXT: ldr w12, [sp, #448]
+; CHECK-GI-NEXT: mul v27.4s, v25.4s, v0.4s
+; CHECK-GI-NEXT: mov v13.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #560]
+; CHECK-GI-NEXT: sxtb w15, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #568]
+; CHECK-GI-NEXT: mov v9.s[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #520]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v20.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #608]
+; CHECK-GI-NEXT: mov v14.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #472]
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: sxtb w13, w13
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v24.s[0], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #560]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mul w8, w8, w11
-; CHECK-GI-NEXT: mov v23.s[3], w10
+; CHECK-GI-NEXT: fmov s10, w10
; CHECK-GI-NEXT: ldr w10, [sp, #552]
-; CHECK-GI-NEXT: ldr w11, [sp, #584]
-; CHECK-GI-NEXT: mov v27.s[0], w12
-; CHECK-GI-NEXT: mov v26.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #616]
-; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: fmov s15, w13
+; CHECK-GI-NEXT: mov v8.s[3], w12
+; CHECK-GI-NEXT: sxtb w12, w14
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v14.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: mov v10.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #576]
+; CHECK-GI-NEXT: mov v9.s[2], w15
+; CHECK-GI-NEXT: mul w8, w8, w10
+; CHECK-GI-NEXT: mov v15.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #512]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v25.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #640]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w12, [sp, #568]
-; CHECK-GI-NEXT: mov v24.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #592]
-; CHECK-GI-NEXT: mov v27.s[1], w11
-; CHECK-GI-NEXT: mov v26.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #624]
-; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: ldr w10, [sp, #584]
+; CHECK-GI-NEXT: ldr w13, [sp, #536]
+; CHECK-GI-NEXT: mla v27.4s, v28.4s, v31.4s
+; CHECK-GI-NEXT: mul v30.4s, v30.4s, v13.4s
+; CHECK-GI-NEXT: mov v10.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #592]
+; CHECK-GI-NEXT: fmov s25, w8
+; CHECK-GI-NEXT: mov v14.s[3], w9
+; CHECK-GI-NEXT: sxtb w9, w12
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w11, [sp, #600]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v28.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #648]
+; CHECK-GI-NEXT: sxtb w8, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #624]
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v9.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #600]
+; CHECK-GI-NEXT: mla v30.4s, v24.4s, v12.4s
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: mov v10.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #632]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #656]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v24.s[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #632]
-; CHECK-GI-NEXT: mov v27.s[2], w10
-; CHECK-GI-NEXT: mov v26.s[2], w9
+; CHECK-GI-NEXT: fmov s28, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #688]
+; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w10, [sp, #656]
+; CHECK-GI-NEXT: mov v15.s[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #544]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: ldr w9, [sp, #664]
-; CHECK-GI-NEXT: mov v28.s[1], w8
+; CHECK-GI-NEXT: mov v28.s[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #696]
+; CHECK-GI-NEXT: fmov s11, w8
+; CHECK-GI-NEXT: fmov s31, w11
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: sxtb w12, w13
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w8, [sp, #680]
-; CHECK-GI-NEXT: mov v24.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #672]
-; CHECK-GI-NEXT: mov v27.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #704]
-; CHECK-GI-NEXT: mov v26.s[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #736]
+; CHECK-GI-NEXT: ldr w11, [sp, #672]
+; CHECK-GI-NEXT: ldr w8, [sp, #616]
+; CHECK-GI-NEXT: mov v11.s[1], w9
+; CHECK-GI-NEXT: mov v15.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #608]
+; CHECK-GI-NEXT: mov v31.s[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #704]
+; CHECK-GI-NEXT: ldr w9, [sp, #640]
+; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mul v24.4s, v26.4s, v14.4s
+; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w9, w9
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v28.s[2], w10
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v29.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #688]
-; CHECK-GI-NEXT: ldr w10, [sp, #696]
-; CHECK-GI-NEXT: mov v31.s[0], w11
+; CHECK-GI-NEXT: mov v11.s[2], w11
; CHECK-GI-NEXT: ldr w11, [sp, #712]
-; CHECK-GI-NEXT: mov v30.s[0], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #744]
+; CHECK-GI-NEXT: mov v0.s[2], w12
+; CHECK-GI-NEXT: mov v31.s[2], w10
+; CHECK-GI-NEXT: ldr w12, [sp, #648]
+; CHECK-GI-NEXT: mov v28.s[2], w9
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: ldr w10, [sp, #720]
+; CHECK-GI-NEXT: ldr w9, [sp, #680]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w14, [sp, #776]
+; CHECK-GI-NEXT: mul v26.4s, v29.4s, v15.4s
+; CHECK-GI-NEXT: mla v24.4s, v22.4s, v8.4s
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v0.s[3], w8
+; CHECK-GI-NEXT: mov v31.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #784]
+; CHECK-GI-NEXT: mov v28.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #752]
+; CHECK-GI-NEXT: fmov s13, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #792]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v28.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #768]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v29.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #720]
-; CHECK-GI-NEXT: mov v31.s[1], w11
+; CHECK-GI-NEXT: mov v11.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #760]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: ldr w8, [sp, #728]
+; CHECK-GI-NEXT: fmov s14, w11
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w11, [sp, #728]
-; CHECK-GI-NEXT: mov v30.s[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #752]
+; CHECK-GI-NEXT: ldr w11, [sp, #744]
+; CHECK-GI-NEXT: fmov s12, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #824]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v8.s[0], w9
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v19.4s
-; CHECK-GI-NEXT: sxtb w9, w13
-; CHECK-GI-NEXT: mov v29.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #760]
-; CHECK-GI-NEXT: mov v31.s[2], w8
-; CHECK-GI-NEXT: sxtb w8, w10
-; CHECK-GI-NEXT: sxtb w10, w14
-; CHECK-GI-NEXT: mov v30.s[2], w9
-; CHECK-GI-NEXT: ldr w14, [sp, #808]
-; CHECK-GI-NEXT: ldr w13, [sp, #784]
-; CHECK-GI-NEXT: mov v8.s[1], w10
+; CHECK-GI-NEXT: mla v26.4s, v23.4s, v9.4s
+; CHECK-GI-NEXT: ldr w13, [sp, #984]
+; CHECK-GI-NEXT: mov v14.s[1], w10
; CHECK-GI-NEXT: sxtb w10, w12
-; CHECK-GI-NEXT: ldr w9, [sp, #792]
-; CHECK-GI-NEXT: sxtb w12, w14
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mul v5.4s, v5.4s, v21.4s
-; CHECK-GI-NEXT: mov v31.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #840]
+; CHECK-GI-NEXT: mov v13.s[1], w8
+; CHECK-GI-NEXT: mov v12.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #832]
+; CHECK-GI-NEXT: ldr w8, [sp, #736]
+; CHECK-GI-NEXT: fmov s29, w10
+; CHECK-GI-NEXT: ldr w12, [sp, #768]
+; CHECK-GI-NEXT: ldr w10, [sp, #800]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v30.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #872]
-; CHECK-GI-NEXT: mov v9.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #816]
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v8.s[2], w13
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w13, [sp, #824]
-; CHECK-GI-NEXT: mov v21.s[0], wzr
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: fmov s6, wzr
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v11.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #848]
-; CHECK-GI-NEXT: mov v10.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #880]
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v9.s[1], w12
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v8.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #904]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w12, [sp, #832]
-; CHECK-GI-NEXT: mov v11.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #856]
-; CHECK-GI-NEXT: mov v29.s[3], w8
-; CHECK-GI-NEXT: mov v10.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #888]
+; CHECK-GI-NEXT: fmov s2, wzr
+; CHECK-GI-NEXT: mov v29.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #840]
+; CHECK-GI-NEXT: mov v13.s[2], w8
+; CHECK-GI-NEXT: mov v12.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #808]
+; CHECK-GI-NEXT: mov v14.s[2], w10
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v9.s[2], w13
+; CHECK-GI-NEXT: ldr w8, [sp, #776]
+; CHECK-GI-NEXT: ldr w10, [sp, #848]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: fmov s5, wzr
+; CHECK-GI-NEXT: fmov s4, wzr
+; CHECK-GI-NEXT: mov v29.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #856]
+; CHECK-GI-NEXT: mov v13.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #864]
+; CHECK-GI-NEXT: mov v14.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #888]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v12.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #912]
+; CHECK-GI-NEXT: fmov s7, wzr
+; CHECK-GI-NEXT: fmov s15, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #920]
+; CHECK-GI-NEXT: mov v12.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #872]
+; CHECK-GI-NEXT: mov v29.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #896]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: fmov s22, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #928]
+; CHECK-GI-NEXT: mov v15.s[1], w11
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v11.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #896]
-; CHECK-GI-NEXT: ldr w13, [sp, #864]
-; CHECK-GI-NEXT: mov v10.s[2], w10
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: fmov s8, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #952]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: ldr w11, [sp, #904]
+; CHECK-GI-NEXT: mov v22.s[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #936]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #920]
-; CHECK-GI-NEXT: mov v9.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #968]
+; CHECK-GI-NEXT: mov v19.s[2], wzr
+; CHECK-GI-NEXT: mov v21.s[2], wzr
+; CHECK-GI-NEXT: mov v15.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #960]
+; CHECK-GI-NEXT: mov v8.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #880]
+; CHECK-GI-NEXT: fmov s23, w9
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v12.s[1], w9
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: ldr w9, [sp, #944]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w8, [sp, #800]
-; CHECK-GI-NEXT: ldr w9, [sp, #928]
-; CHECK-GI-NEXT: mov v10.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1032]
-; CHECK-GI-NEXT: mov v11.s[3], w13
-; CHECK-GI-NEXT: mov v14.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #976]
-; CHECK-GI-NEXT: ldr w13, [sp, #936]
-; CHECK-GI-NEXT: mov v12.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1000]
+; CHECK-GI-NEXT: mov v22.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #912]
+; CHECK-GI-NEXT: mov v8.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #968]
+; CHECK-GI-NEXT: mov v23.s[1], w8
+; CHECK-GI-NEXT: mov v15.s[3], w12
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: sxtb w12, w13
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: add v18.4s, v18.4s, v20.4s
+; CHECK-GI-NEXT: mov v22.s[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: fmov s9, w12
+; CHECK-GI-NEXT: mov v23.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1048]
+; CHECK-GI-NEXT: ldr w12, [sp, #1056]
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v15.4s
+; CHECK-GI-NEXT: sxtb w13, w11
+; CHECK-GI-NEXT: mov v8.s[3], w9
+; CHECK-GI-NEXT: sxtb w11, w10
+; CHECK-GI-NEXT: ldr w9, [sp, #1000]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v21.s[1], wzr
-; CHECK-GI-NEXT: mov v15.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1040]
+; CHECK-GI-NEXT: mov v9.s[1], w13
+; CHECK-GI-NEXT: ldr w10, [sp, #1016]
+; CHECK-GI-NEXT: ldr w8, [sp, #816]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: mov v16.s[1], wzr
+; CHECK-GI-NEXT: mla v0.4s, v10.4s, v29.4s
+; CHECK-GI-NEXT: fmov s10, w11
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v14.s[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #984]
-; CHECK-GI-NEXT: mov v13.s[0], w13
-; CHECK-GI-NEXT: mov v19.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1008]
+; CHECK-GI-NEXT: ldr w11, [sp, #1024]
+; CHECK-GI-NEXT: mul v20.4s, v11.4s, v8.4s
+; CHECK-GI-NEXT: ldr q8, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v9.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1008]
+; CHECK-GI-NEXT: fmov s29, w10
+; CHECK-GI-NEXT: mov v10.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1064]
; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v17.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], wzr
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w13, [sp, #944]
+; CHECK-GI-NEXT: mov v6.s[1], wzr
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v5.s[1], wzr
+; CHECK-GI-NEXT: mov v4.s[1], wzr
+; CHECK-GI-NEXT: mov v7.s[1], wzr
+; CHECK-GI-NEXT: mov v10.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1080]
+; CHECK-GI-NEXT: mov v8.s[1], wzr
+; CHECK-GI-NEXT: mov v9.s[3], w9
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v15.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1048]
+; CHECK-GI-NEXT: ldr w10, [sp, #1032]
+; CHECK-GI-NEXT: sxtb w9, w12
+; CHECK-GI-NEXT: mov v29.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1072]
+; CHECK-GI-NEXT: mov v19.s[3], wzr
+; CHECK-GI-NEXT: mov v21.s[3], wzr
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mul w8, w8, w9
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v14.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1064]
-; CHECK-GI-NEXT: sxtb w13, w13
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v19.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #992]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v13.s[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #952]
-; CHECK-GI-NEXT: mov v15.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1016]
+; CHECK-GI-NEXT: mov v16.s[2], wzr
+; CHECK-GI-NEXT: mov v17.s[2], wzr
+; CHECK-GI-NEXT: mov v3.s[2], wzr
+; CHECK-GI-NEXT: mov v6.s[2], wzr
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v5.s[2], wzr
+; CHECK-GI-NEXT: mov v4.s[2], wzr
+; CHECK-GI-NEXT: mov v7.s[2], wzr
+; CHECK-GI-NEXT: mov v8.s[2], wzr
+; CHECK-GI-NEXT: mov v29.s[2], w10
+; CHECK-GI-NEXT: mov v10.s[3], w11
+; CHECK-GI-NEXT: add v19.4s, v19.4s, v21.4s
+; CHECK-GI-NEXT: ldr w9, [sp, #976]
+; CHECK-GI-NEXT: fmov s21, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1040]
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[3], wzr
+; CHECK-GI-NEXT: mov v17.s[3], wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mul w8, w8, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1056]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w13, w13
-; CHECK-GI-NEXT: mov v12.s[3], w9
-; CHECK-GI-NEXT: sxtb w9, w11
-; CHECK-GI-NEXT: mov v14.s[3], w10
-; CHECK-GI-NEXT: mov v21.s[2], wzr
-; CHECK-GI-NEXT: sxtb w10, w12
-; CHECK-GI-NEXT: mul v6.4s, v6.4s, v22.4s
-; CHECK-GI-NEXT: mov v22.s[0], w8
-; CHECK-GI-NEXT: mov v13.s[2], w13
-; CHECK-GI-NEXT: mov v19.s[2], w9
-; CHECK-GI-NEXT: ldr w8, [sp, #960]
-; CHECK-GI-NEXT: mov v15.s[3], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #1024]
-; CHECK-GI-NEXT: mov v25.s[1], wzr
-; CHECK-GI-NEXT: mul v7.4s, v7.4s, v23.4s
-; CHECK-GI-NEXT: mov v21.s[3], wzr
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mul v23.4s, v27.4s, v11.4s
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v22.s[1], wzr
-; CHECK-GI-NEXT: mul v27.4s, v28.4s, v12.4s
-; CHECK-GI-NEXT: mul v28.4s, v31.4s, v14.4s
-; CHECK-GI-NEXT: mul v31.4s, v8.4s, v15.4s
-; CHECK-GI-NEXT: mov v13.s[3], w8
-; CHECK-GI-NEXT: mov v19.s[3], w9
-; CHECK-GI-NEXT: mla v3.4s, v0.4s, v16.4s
+; CHECK-GI-NEXT: mov v11.16b, v8.16b
+; CHECK-GI-NEXT: mov v3.s[3], wzr
+; CHECK-GI-NEXT: mov v6.s[3], wzr
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mov v5.s[3], wzr
+; CHECK-GI-NEXT: mov v4.s[3], wzr
+; CHECK-GI-NEXT: mov v7.s[3], wzr
+; CHECK-GI-NEXT: mov v25.s[1], wzr
+; CHECK-GI-NEXT: mov v21.s[1], wzr
+; CHECK-GI-NEXT: mul v8.4s, v13.4s, v9.4s
+; CHECK-GI-NEXT: mul v9.4s, v14.4s, v10.4s
+; CHECK-GI-NEXT: mov v23.s[3], w9
+; CHECK-GI-NEXT: mov v29.s[3], w8
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v11.s[3], wzr
+; CHECK-GI-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: add v3.4s, v3.4s, v6.4s
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT: add v4.4s, v4.4s, v7.4s
; CHECK-GI-NEXT: mov v25.s[2], wzr
-; CHECK-GI-NEXT: add v0.4s, v21.4s, v21.4s
-; CHECK-GI-NEXT: mla v5.4s, v1.4s, v17.4s
-; CHECK-GI-NEXT: mla v6.4s, v2.4s, v18.4s
-; CHECK-GI-NEXT: mov v22.s[2], wzr
-; CHECK-GI-NEXT: mla v7.4s, v4.4s, v20.4s
-; CHECK-GI-NEXT: mla v23.4s, v24.4s, v9.4s
-; CHECK-GI-NEXT: mla v27.4s, v26.4s, v10.4s
-; CHECK-GI-NEXT: mla v28.4s, v29.4s, v13.4s
-; CHECK-GI-NEXT: mla v31.4s, v30.4s, v19.4s
-; CHECK-GI-NEXT: add v1.4s, v21.4s, v0.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: mov v21.s[2], wzr
+; CHECK-GI-NEXT: mla v20.4s, v28.4s, v22.4s
+; CHECK-GI-NEXT: mla v8.4s, v31.4s, v23.4s
+; CHECK-GI-NEXT: mla v9.4s, v12.4s, v29.4s
+; CHECK-GI-NEXT: add v5.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v18.4s
+; CHECK-GI-NEXT: add v3.4s, v11.4s, v3.4s
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: add v4.4s, v27.4s, v30.4s
+; CHECK-GI-NEXT: add v6.4s, v24.4s, v26.4s
+; CHECK-GI-NEXT: ldr x29, [sp, #80] // 8-byte Folded Reload
; CHECK-GI-NEXT: mov v25.s[3], wzr
-; CHECK-GI-NEXT: add v2.4s, v3.4s, v5.4s
-; CHECK-GI-NEXT: mov v22.s[3], wzr
-; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v23.4s, v27.4s
-; CHECK-GI-NEXT: add v4.4s, v28.4s, v31.4s
-; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: add v3.4s, v25.4s, v0.4s
-; CHECK-GI-NEXT: add v0.4s, v22.4s, v0.4s
-; CHECK-GI-NEXT: add v1.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: addv s1, v2.4s
+; CHECK-GI-NEXT: mov v21.s[3], wzr
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v20.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-GI-NEXT: add v5.4s, v8.4s, v9.4s
+; CHECK-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: add v3.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v1.4s, v25.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s
+; CHECK-GI-NEXT: add v2.4s, v21.4s, v2.4s
+; CHECK-GI-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
; CHECK-GI-NEXT: addv s0, v0.4s
; CHECK-GI-NEXT: fmov w8, s1
; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-GI-NEXT: add sp, sp, #96
; CHECK-GI-NEXT: ret
entry:
%az = sext <33 x i8> %a to <33 x i32>
@@ -5350,238 +5495,309 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
;
; CHECK-GI-LABEL: test_sdot_v33i8_double_nomla:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: stp d13, d12, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: str x29, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 64
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: sxtb w10, w0
-; CHECK-GI-NEXT: ldr w9, [sp, #16]
-; CHECK-GI-NEXT: ldr w8, [sp, #48]
-; CHECK-GI-NEXT: sxtb w11, w4
-; CHECK-GI-NEXT: sxtb w12, w6
-; CHECK-GI-NEXT: ldr w13, [sp, #592]
-; CHECK-GI-NEXT: mov v0.s[0], w10
-; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: .cfi_offset b8, -24
+; CHECK-GI-NEXT: .cfi_offset b9, -32
+; CHECK-GI-NEXT: .cfi_offset b10, -40
+; CHECK-GI-NEXT: .cfi_offset b11, -48
+; CHECK-GI-NEXT: .cfi_offset b12, -56
+; CHECK-GI-NEXT: .cfi_offset b13, -64
+; CHECK-GI-NEXT: sxtb w8, w0
+; CHECK-GI-NEXT: sxtb w9, w1
+; CHECK-GI-NEXT: sxtb w12, w4
+; CHECK-GI-NEXT: ldr w10, [sp, #72]
+; CHECK-GI-NEXT: ldr w11, [sp, #96]
+; CHECK-GI-NEXT: sxtb w13, w5
+; CHECK-GI-NEXT: fmov s22, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #64]
+; CHECK-GI-NEXT: fmov s23, w12
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w12, w7
+; CHECK-GI-NEXT: fmov s18, wzr
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v1.s[0], w11
-; CHECK-GI-NEXT: sxtb w10, w1
-; CHECK-GI-NEXT: sxtb w11, w5
-; CHECK-GI-NEXT: mov v2.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #24]
-; CHECK-GI-NEXT: mov v3.s[0], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #56]
-; CHECK-GI-NEXT: mov v22.s[0], wzr
-; CHECK-GI-NEXT: mov v0.s[1], w10
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #80]
+; CHECK-GI-NEXT: fmov s19, wzr
+; CHECK-GI-NEXT: fmov s20, wzr
+; CHECK-GI-NEXT: mov v22.s[1], w9
+; CHECK-GI-NEXT: sxtb w9, w2
+; CHECK-GI-NEXT: mov v23.s[1], w13
+; CHECK-GI-NEXT: fmov s24, w8
+; CHECK-GI-NEXT: sxtb w8, w11
+; CHECK-GI-NEXT: sxtb w11, w6
+; CHECK-GI-NEXT: ldr w13, [sp, #232]
+; CHECK-GI-NEXT: mov v18.s[1], wzr
+; CHECK-GI-NEXT: mov v19.s[1], wzr
+; CHECK-GI-NEXT: fmov s25, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #80]
+; CHECK-GI-NEXT: fmov s21, wzr
+; CHECK-GI-NEXT: mov v22.s[2], w9
+; CHECK-GI-NEXT: mov v24.s[1], w10
+; CHECK-GI-NEXT: sxtb w10, w3
+; CHECK-GI-NEXT: ldr w9, [sp, #104]
+; CHECK-GI-NEXT: mov v23.s[2], w11
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v1.s[1], w11
-; CHECK-GI-NEXT: sxtb w11, w2
-; CHECK-GI-NEXT: mov v2.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #32]
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v3.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #64]
-; CHECK-GI-NEXT: mov v22.s[1], wzr
-; CHECK-GI-NEXT: mov v0.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #136]
+; CHECK-GI-NEXT: mov v18.s[2], wzr
+; CHECK-GI-NEXT: mov v19.s[2], wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: sxtb w11, w3
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v1.s[2], w12
-; CHECK-GI-NEXT: mov v4.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #40]
-; CHECK-GI-NEXT: mov v2.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #88]
-; CHECK-GI-NEXT: sxtb w12, w7
-; CHECK-GI-NEXT: mov v3.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #72]
-; CHECK-GI-NEXT: mov v0.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #112]
+; CHECK-GI-NEXT: fmov s6, wzr
+; CHECK-GI-NEXT: fmov s7, wzr
+; CHECK-GI-NEXT: mov v22.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #128]
+; CHECK-GI-NEXT: mov v24.s[2], w8
+; CHECK-GI-NEXT: mov v25.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #112]
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: sxtb w11, w11
+; CHECK-GI-NEXT: mov v23.s[3], w12
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v1.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #144]
+; CHECK-GI-NEXT: ldr w12, [sp, #160]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v2.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #96]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #176]
+; CHECK-GI-NEXT: fmov s26, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #144]
+; CHECK-GI-NEXT: mov v18.s[3], wzr
+; CHECK-GI-NEXT: mov v25.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #120]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v3.s[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #120]
-; CHECK-GI-NEXT: mov v5.s[0], w11
+; CHECK-GI-NEXT: mov v24.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #168]
; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v26.s[1], w11
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: ldr w11, [sp, #152]
-; CHECK-GI-NEXT: mov v6.s[0], w12
+; CHECK-GI-NEXT: fmov s27, w12
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w12, [sp, #104]
-; CHECK-GI-NEXT: mov v4.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #184]
-; CHECK-GI-NEXT: mov v7.s[0], w9
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #128]
-; CHECK-GI-NEXT: mov v5.s[1], w8
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: mov v6.s[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #480]
+; CHECK-GI-NEXT: ldr w12, [sp, #224]
+; CHECK-GI-NEXT: ldr w11, [sp, #152]
+; CHECK-GI-NEXT: mov v25.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #192]
+; CHECK-GI-NEXT: add v22.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: mov v27.s[1], w8
+; CHECK-GI-NEXT: mov v19.s[3], wzr
+; CHECK-GI-NEXT: fmov s5, wzr
+; CHECK-GI-NEXT: mov v26.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #200]
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v4.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #192]
-; CHECK-GI-NEXT: mov v7.s[1], w10
+; CHECK-GI-NEXT: fmov s16, wzr
+; CHECK-GI-NEXT: fmov s17, wzr
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: sxtb w8, w10
+; CHECK-GI-NEXT: sxtb w10, w12
+; CHECK-GI-NEXT: fmov s28, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #176]
+; CHECK-GI-NEXT: sxtb w12, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #208]
+; CHECK-GI-NEXT: fmov s29, w10
+; CHECK-GI-NEXT: sxtb w10, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #536]
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v28.s[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #184]
+; CHECK-GI-NEXT: mov v26.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #528]
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: mov v29.s[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #240]
+; CHECK-GI-NEXT: mov v27.s[2], w9
+; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w10, [sp, #136]
-; CHECK-GI-NEXT: mov v5.s[2], w9
-; CHECK-GI-NEXT: sxtb w9, w11
+; CHECK-GI-NEXT: ldr w9, [sp, #216]
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w11, [sp, #168]
-; CHECK-GI-NEXT: mov v6.s[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #488]
-; CHECK-GI-NEXT: mov v16.s[0], w9
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: ldr w9, [sp, #200]
-; CHECK-GI-NEXT: mov v7.s[2], w12
+; CHECK-GI-NEXT: mov v28.s[2], w13
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w12, [sp, #512]
-; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v5.s[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #544]
+; CHECK-GI-NEXT: fmov s30, w10
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v6.s[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #496]
+; CHECK-GI-NEXT: ldr w10, [sp, #256]
+; CHECK-GI-NEXT: mov v29.s[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #248]
+; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #560]
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: add v23.4s, v24.4s, v25.4s
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v16.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #208]
-; CHECK-GI-NEXT: mov v7.s[3], w9
-; CHECK-GI-NEXT: sxtb w9, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #520]
+; CHECK-GI-NEXT: mov v30.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #544]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v28.s[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #568]
+; CHECK-GI-NEXT: mov v29.s[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #592]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v17.s[0], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #552]
+; CHECK-GI-NEXT: fmov s31, w8
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: ldr w8, [sp, #552]
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v30.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #600]
+; CHECK-GI-NEXT: fmov s8, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #624]
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v19.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #504]
+; CHECK-GI-NEXT: mov v31.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #576]
+; CHECK-GI-NEXT: fmov s9, w12
+; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v16.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #576]
-; CHECK-GI-NEXT: mov v18.s[0], w8
-; CHECK-GI-NEXT: sxtb w8, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #528]
+; CHECK-GI-NEXT: fmov s1, wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v17.s[1], w10
+; CHECK-GI-NEXT: mov v30.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #632]
+; CHECK-GI-NEXT: mov v9.s[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #608]
+; CHECK-GI-NEXT: fmov s10, w10
+; CHECK-GI-NEXT: mov v31.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #656]
+; CHECK-GI-NEXT: ldr w10, [sp, #688]
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: ldr w10, [sp, #560]
-; CHECK-GI-NEXT: mov v19.s[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #608]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v16.s[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #584]
-; CHECK-GI-NEXT: mov v20.s[0], w11
-; CHECK-GI-NEXT: sxtb w10, w10
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: ldr w11, [sp, #536]
-; CHECK-GI-NEXT: mov v17.s[2], w12
+; CHECK-GI-NEXT: fmov s3, wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: mov v19.s[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #616]
-; CHECK-GI-NEXT: mov v21.s[0], w8
-; CHECK-GI-NEXT: ldr w12, [sp, #568]
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v22.s[2], wzr
-; CHECK-GI-NEXT: mov v20.s[1], w9
-; CHECK-GI-NEXT: sxtb w9, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #640]
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v17.s[3], w11
-; CHECK-GI-NEXT: sxtb w11, w13
-; CHECK-GI-NEXT: mov v21.s[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #672]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v19.s[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #624]
-; CHECK-GI-NEXT: ldr w8, [sp, #600]
-; CHECK-GI-NEXT: mov v20.s[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #704]
-; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v23.s[0], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #648]
+; CHECK-GI-NEXT: fmov s2, wzr
+; CHECK-GI-NEXT: mov v9.s[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #664]
+; CHECK-GI-NEXT: mov v10.s[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #696]
+; CHECK-GI-NEXT: fmov s11, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #640]
+; CHECK-GI-NEXT: fmov s12, w10
; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v24.s[0], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #736]
-; CHECK-GI-NEXT: mov v21.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #680]
+; CHECK-GI-NEXT: ldr w10, [sp, #672]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: fmov s4, wzr
+; CHECK-GI-NEXT: mov v11.s[1], w11
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v25.s[0], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #712]
+; CHECK-GI-NEXT: mov v20.s[1], wzr
+; CHECK-GI-NEXT: mov v12.s[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #704]
+; CHECK-GI-NEXT: mov v10.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #648]
+; CHECK-GI-NEXT: mov v21.s[1], wzr
+; CHECK-GI-NEXT: mov v6.s[1], wzr
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v7.s[1], wzr
+; CHECK-GI-NEXT: mov v5.s[1], wzr
; CHECK-GI-NEXT: sxtb w9, w9
-; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: mov v23.s[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #656]
-; CHECK-GI-NEXT: sxtb w11, w11
-; CHECK-GI-NEXT: mov v26.s[0], w9
+; CHECK-GI-NEXT: mov v11.s[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #680]
+; CHECK-GI-NEXT: mov v12.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #712]
+; CHECK-GI-NEXT: mov v16.s[1], wzr
+; CHECK-GI-NEXT: mov v10.s[3], w9
; CHECK-GI-NEXT: ldr w9, [sp, #720]
-; CHECK-GI-NEXT: mov v24.s[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #688]
; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: mov v25.s[1], w11
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v17.s[1], wzr
+; CHECK-GI-NEXT: add v19.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v11.s[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #728]
+; CHECK-GI-NEXT: mov v12.s[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #752]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: fmov s13, w9
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: ldr w9, [sp, #760]
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mov v3.s[1], wzr
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v4.s[1], wzr
+; CHECK-GI-NEXT: mov v13.s[1], w10
+; CHECK-GI-NEXT: fmov s24, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #736]
+; CHECK-GI-NEXT: mov v20.s[2], wzr
+; CHECK-GI-NEXT: mov v21.s[2], wzr
+; CHECK-GI-NEXT: mov v6.s[2], wzr
+; CHECK-GI-NEXT: sxtb w8, w8
+; CHECK-GI-NEXT: mov v7.s[2], wzr
+; CHECK-GI-NEXT: mov v8.s[1], wzr
+; CHECK-GI-NEXT: mov v24.s[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #768]
+; CHECK-GI-NEXT: ldr w12, [sp, #584]
+; CHECK-GI-NEXT: mov v13.s[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #784]
+; CHECK-GI-NEXT: ldr w11, [sp, #616]
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v5.s[2], wzr
+; CHECK-GI-NEXT: mov v16.s[2], wzr
+; CHECK-GI-NEXT: sxtb w10, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: mov v17.s[2], wzr
+; CHECK-GI-NEXT: mov v24.s[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #776]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: fmov s18, w10
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v3.s[2], wzr
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v4.s[2], wzr
+; CHECK-GI-NEXT: mov v20.s[3], wzr
+; CHECK-GI-NEXT: mov v21.s[3], wzr
+; CHECK-GI-NEXT: mov v6.s[3], wzr
+; CHECK-GI-NEXT: mov v7.s[3], wzr
; CHECK-GI-NEXT: mov v18.s[1], wzr
; CHECK-GI-NEXT: sxtb w12, w12
-; CHECK-GI-NEXT: ldr w11, [sp, #632]
-; CHECK-GI-NEXT: mov v23.s[2], w10
-; CHECK-GI-NEXT: mov v26.s[1], wzr
-; CHECK-GI-NEXT: ldr w10, [sp, #664]
+; CHECK-GI-NEXT: sxtb w11, w11
; CHECK-GI-NEXT: sxtb w8, w8
-; CHECK-GI-NEXT: mov v24.s[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #696]
-; CHECK-GI-NEXT: mov v22.s[3], wzr
-; CHECK-GI-NEXT: mov v25.s[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #728]
-; CHECK-GI-NEXT: mov v20.s[3], w8
-; CHECK-GI-NEXT: sxtb w8, w11
-; CHECK-GI-NEXT: sxtb w10, w10
-; CHECK-GI-NEXT: sxtb w11, w12
; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov v8.s[2], wzr
+; CHECK-GI-NEXT: mov v31.s[3], w12
+; CHECK-GI-NEXT: mov v9.s[3], w11
+; CHECK-GI-NEXT: mov v5.s[3], wzr
+; CHECK-GI-NEXT: mov v16.s[3], wzr
+; CHECK-GI-NEXT: mov v17.s[3], wzr
+; CHECK-GI-NEXT: mov v13.s[3], w8
; CHECK-GI-NEXT: mov v18.s[2], wzr
-; CHECK-GI-NEXT: mov v26.s[2], wzr
-; CHECK-GI-NEXT: mov v21.s[3], w8
-; CHECK-GI-NEXT: mov v23.s[3], w10
-; CHECK-GI-NEXT: mov v24.s[3], w11
-; CHECK-GI-NEXT: mov v25.s[3], w9
-; CHECK-GI-NEXT: add v27.4s, v22.4s, v22.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mov v24.s[3], w9
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v3.s[3], wzr
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mov v4.s[3], wzr
+; CHECK-GI-NEXT: add v20.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: add v6.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mov v8.s[3], wzr
+; CHECK-GI-NEXT: add v25.4s, v26.4s, v27.4s
+; CHECK-GI-NEXT: add v26.4s, v28.4s, v29.4s
; CHECK-GI-NEXT: mov v18.s[3], wzr
-; CHECK-GI-NEXT: mov v26.s[3], wzr
-; CHECK-GI-NEXT: add v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT: add v5.4s, v22.4s, v27.4s
-; CHECK-GI-NEXT: add v6.4s, v19.4s, v20.4s
-; CHECK-GI-NEXT: add v7.4s, v21.4s, v23.4s
-; CHECK-GI-NEXT: add v16.4s, v24.4s, v25.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v3.4s, v27.4s, v27.4s
-; CHECK-GI-NEXT: add v2.4s, v18.4s, v5.4s
-; CHECK-GI-NEXT: add v4.4s, v4.4s, v6.4s
-; CHECK-GI-NEXT: add v5.4s, v26.4s, v5.4s
-; CHECK-GI-NEXT: add v6.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: add v3.4s, v5.4s, v3.4s
-; CHECK-GI-NEXT: add v2.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT: add v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: add v7.4s, v30.4s, v31.4s
+; CHECK-GI-NEXT: add v5.4s, v5.4s, v20.4s
+; CHECK-GI-NEXT: add v17.4s, v9.4s, v10.4s
+; CHECK-GI-NEXT: add v20.4s, v11.4s, v12.4s
+; CHECK-GI-NEXT: add v21.4s, v13.4s, v24.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v6.4s
+; CHECK-GI-NEXT: add v3.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: add v4.4s, v25.4s, v26.4s
+; CHECK-GI-NEXT: add v5.4s, v8.4s, v5.4s
+; CHECK-GI-NEXT: add v6.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT: add v7.4s, v7.4s, v17.4s
+; CHECK-GI-NEXT: add v16.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: ldr x29, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT: add v0.4s, v18.4s, v0.4s
+; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT: add v3.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v4.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT: ldp d11, d10, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v4.4s, v0.4s
; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ldp d13, d12, [sp], #64 // 16-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%az = sext <33 x i8> %a to <33 x i32>
@@ -5610,15 +5826,15 @@ define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_udot_v48i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], wzr
+; CHECK-GI-NEXT: fmov s0, wzr
; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
; CHECK-GI-NEXT: ldr q7, [x0, #32]
; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
; CHECK-GI-NEXT: ldr q17, [x1, #32]
; CHECK-GI-NEXT: ldp q4, q5, [x0]
-; CHECK-GI-NEXT: ldp q6, q16, [x1]
; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: ldp q6, q16, [x1]
; CHECK-GI-NEXT: udot v2.4s, v17.16b, v7.16b
; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b
; CHECK-GI-NEXT: udot v3.4s, v16.16b, v5.16b
@@ -5658,7 +5874,7 @@ define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
;
; CHECK-GI-LABEL: test_udot_v48i8_nomla:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], wzr
+; CHECK-GI-NEXT: fmov s0, wzr
; CHECK-GI-NEXT: movi v1.16b, #1
; CHECK-GI-NEXT: ldr q7, [x0, #32]
; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
@@ -5701,15 +5917,15 @@ define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b
;
; CHECK-GI-LABEL: test_sdot_v48i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], wzr
+; CHECK-GI-NEXT: fmov s0, wzr
; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
; CHECK-GI-NEXT: ldr q7, [x0, #32]
; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
; CHECK-GI-NEXT: ldr q17, [x1, #32]
; CHECK-GI-NEXT: ldp q4, q5, [x0]
-; CHECK-GI-NEXT: ldp q6, q16, [x1]
; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: ldp q6, q16, [x1]
; CHECK-GI-NEXT: sdot v2.4s, v17.16b, v7.16b
; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b
; CHECK-GI-NEXT: sdot v3.4s, v16.16b, v5.16b
@@ -6123,408 +6339,412 @@ define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w11, [sp, #80]
-; CHECK-GI-NEXT: ldr w8, [sp, #208]
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w12, [sp, #336]
+; CHECK-GI-NEXT: ldr w10, [sp, #80]
; CHECK-GI-NEXT: ldr w9, [sp, #88]
-; CHECK-GI-NEXT: mov v20.s[0], wzr
-; CHECK-GI-NEXT: fmov s1, w11
-; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: ldr w11, [sp, #464]
-; CHECK-GI-NEXT: ldr w8, [sp, #592]
-; CHECK-GI-NEXT: ldr w10, [sp, #216]
-; CHECK-GI-NEXT: fmov s3, w12
+; CHECK-GI-NEXT: fmov s2, w0
+; CHECK-GI-NEXT: ldr w11, [sp, #208]
+; CHECK-GI-NEXT: ldr w8, [sp, #216]
+; CHECK-GI-NEXT: fmov s1, wzr
+; CHECK-GI-NEXT: fmov s3, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #336]
+; CHECK-GI-NEXT: ldr w12, [sp, #720]
; CHECK-GI-NEXT: fmov s4, w11
-; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: ldr w11, [sp, #600]
-; CHECK-GI-NEXT: fmov s5, w8
-; CHECK-GI-NEXT: mov v1.b[1], w9
-; CHECK-GI-NEXT: mov v2.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #344]
-; CHECK-GI-NEXT: ldr w9, [sp, #472]
-; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: ldr w12, [sp, #848]
-; CHECK-GI-NEXT: ldr w13, [sp, #728]
-; CHECK-GI-NEXT: mov v20.s[1], wzr
-; CHECK-GI-NEXT: mov v3.b[1], w10
-; CHECK-GI-NEXT: mov v4.b[1], w9
+; CHECK-GI-NEXT: mov v2.b[1], w1
+; CHECK-GI-NEXT: ldr w11, [sp, #344]
+; CHECK-GI-NEXT: fmov s5, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #592]
+; CHECK-GI-NEXT: fmov s16, w12
+; CHECK-GI-NEXT: mov v3.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #464]
+; CHECK-GI-NEXT: ldr w12, [sp, #1112]
+; CHECK-GI-NEXT: fmov s7, w10
+; CHECK-GI-NEXT: mov v4.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #472]
+; CHECK-GI-NEXT: fmov s6, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #600]
; CHECK-GI-NEXT: mov v5.b[1], w11
-; CHECK-GI-NEXT: ldr w9, [sp, #224]
-; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: ldr w10, [sp, #352]
-; CHECK-GI-NEXT: ldr w11, [sp, #480]
-; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #608]
-; CHECK-GI-NEXT: mov v2.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #104]
-; CHECK-GI-NEXT: fmov s7, w12
-; CHECK-GI-NEXT: mov v3.b[2], w10
-; CHECK-GI-NEXT: mov v4.b[2], w11
+; CHECK-GI-NEXT: ldr w10, [sp, #224]
+; CHECK-GI-NEXT: mov v2.b[2], w2
+; CHECK-GI-NEXT: ldr w11, [sp, #16]
+; CHECK-GI-NEXT: mov v7.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: mov v6.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #96]
+; CHECK-GI-NEXT: mov v4.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #608]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v3.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #352]
+; CHECK-GI-NEXT: mov v2.b[3], w3
+; CHECK-GI-NEXT: mov v7.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #488]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
; CHECK-GI-NEXT: mov v5.b[2], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #232]
-; CHECK-GI-NEXT: mov v0.b[3], w3
+; CHECK-GI-NEXT: mov v6.b[2], w9
+; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: ldr w9, [sp, #232]
+; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v3.b[3], w8
; CHECK-GI-NEXT: ldr w8, [sp, #360]
-; CHECK-GI-NEXT: ldr w11, [sp, #488]
-; CHECK-GI-NEXT: mov v1.b[3], w9
+; CHECK-GI-NEXT: mov v2.b[4], w4
+; CHECK-GI-NEXT: mov v4.b[3], w9
; CHECK-GI-NEXT: ldr w9, [sp, #616]
-; CHECK-GI-NEXT: mov v2.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #240]
-; CHECK-GI-NEXT: ldr w12, [sp, #1232]
-; CHECK-GI-NEXT: mov v3.b[3], w8
-; CHECK-GI-NEXT: mov v4.b[3], w11
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v5.b[3], w8
+; CHECK-GI-NEXT: mov v6.b[3], w10
; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: mov v5.b[3], w9
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: ldr w9, [sp, #368]
-; CHECK-GI-NEXT: ldr w11, [sp, #496]
-; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #624]
-; CHECK-GI-NEXT: mov v2.b[4], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #248]
-; CHECK-GI-NEXT: fmov s18, w12
-; CHECK-GI-NEXT: mov v3.b[4], w9
-; CHECK-GI-NEXT: mov v4.b[4], w11
-; CHECK-GI-NEXT: ldr w9, [sp, #120]
+; CHECK-GI-NEXT: ldr w10, [sp, #240]
+; CHECK-GI-NEXT: mov v7.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #496]
+; CHECK-GI-NEXT: mov v3.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #368]
+; CHECK-GI-NEXT: mov v2.b[5], w5
+; CHECK-GI-NEXT: mov v4.b[4], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #624]
+; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
; CHECK-GI-NEXT: mov v5.b[4], w8
-; CHECK-GI-NEXT: mov v0.b[5], w5
+; CHECK-GI-NEXT: mov v6.b[4], w9
+; CHECK-GI-NEXT: ldr w8, [sp, #120]
+; CHECK-GI-NEXT: ldr w9, [sp, #248]
+; CHECK-GI-NEXT: mov v7.b[4], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #504]
+; CHECK-GI-NEXT: mov v3.b[5], w8
; CHECK-GI-NEXT: ldr w8, [sp, #376]
-; CHECK-GI-NEXT: ldr w11, [sp, #504]
-; CHECK-GI-NEXT: mov v1.b[5], w9
+; CHECK-GI-NEXT: mov v2.b[6], w6
+; CHECK-GI-NEXT: mov v4.b[5], w9
; CHECK-GI-NEXT: ldr w9, [sp, #632]
-; CHECK-GI-NEXT: mov v2.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #256]
-; CHECK-GI-NEXT: ldr w12, [sp, #744]
-; CHECK-GI-NEXT: mov v3.b[5], w8
-; CHECK-GI-NEXT: mov v4.b[5], w11
+; CHECK-GI-NEXT: movi v26.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v5.b[5], w8
+; CHECK-GI-NEXT: mov v6.b[5], w10
; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: mov v5.b[5], w9
-; CHECK-GI-NEXT: mov v0.b[6], w6
-; CHECK-GI-NEXT: ldr w9, [sp, #384]
-; CHECK-GI-NEXT: ldr w11, [sp, #512]
-; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #640]
-; CHECK-GI-NEXT: mov v2.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #264]
-; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v3.b[6], w9
-; CHECK-GI-NEXT: mov v4.b[6], w11
-; CHECK-GI-NEXT: ldr w9, [sp, #136]
+; CHECK-GI-NEXT: ldr w10, [sp, #256]
+; CHECK-GI-NEXT: mov v7.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #512]
+; CHECK-GI-NEXT: mov v3.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #384]
+; CHECK-GI-NEXT: mov v2.b[7], w7
+; CHECK-GI-NEXT: mov v4.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #640]
+; CHECK-GI-NEXT: movi v27.2d, #0000000000000000
; CHECK-GI-NEXT: mov v5.b[6], w8
-; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: ldr w8, [sp, #392]
-; CHECK-GI-NEXT: ldr w11, [sp, #520]
-; CHECK-GI-NEXT: mov v1.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #648]
-; CHECK-GI-NEXT: mov v2.b[7], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #16]
-; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr w8, [sp, #136]
+; CHECK-GI-NEXT: mov v6.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #264]
+; CHECK-GI-NEXT: mov v7.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #520]
; CHECK-GI-NEXT: mov v3.b[7], w8
-; CHECK-GI-NEXT: mov v4.b[7], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #144]
-; CHECK-GI-NEXT: mov v5.b[7], w9
-; CHECK-GI-NEXT: ldr w11, [sp, #272]
-; CHECK-GI-NEXT: mov v0.b[8], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #400]
-; CHECK-GI-NEXT: ldr w10, [sp, #528]
-; CHECK-GI-NEXT: mov v1.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #656]
+; CHECK-GI-NEXT: ldr w8, [sp, #392]
; CHECK-GI-NEXT: mov v2.b[8], w11
+; CHECK-GI-NEXT: mov v4.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #648]
; CHECK-GI-NEXT: ldr w11, [sp, #24]
-; CHECK-GI-NEXT: mov v3.b[8], w9
+; CHECK-GI-NEXT: mov v5.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #144]
+; CHECK-GI-NEXT: mov v6.b[7], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #272]
+; CHECK-GI-NEXT: mov v7.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #528]
+; CHECK-GI-NEXT: mov v3.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #400]
+; CHECK-GI-NEXT: mov v2.b[9], w11
; CHECK-GI-NEXT: mov v4.b[8], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #152]
+; CHECK-GI-NEXT: ldr w10, [sp, #656]
+; CHECK-GI-NEXT: ldr w11, [sp, #32]
; CHECK-GI-NEXT: mov v5.b[8], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #280]
-; CHECK-GI-NEXT: mov v0.b[9], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #408]
-; CHECK-GI-NEXT: ldr w11, [sp, #536]
-; CHECK-GI-NEXT: mov v1.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #664]
-; CHECK-GI-NEXT: mov v2.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #32]
+; CHECK-GI-NEXT: ldr w8, [sp, #152]
+; CHECK-GI-NEXT: mov v6.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #280]
+; CHECK-GI-NEXT: mov v7.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #536]
; CHECK-GI-NEXT: mov v3.b[9], w8
-; CHECK-GI-NEXT: mov v4.b[9], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: mov v5.b[9], w9
-; CHECK-GI-NEXT: ldr w11, [sp, #288]
-; CHECK-GI-NEXT: mov v0.b[10], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #416]
-; CHECK-GI-NEXT: ldr w10, [sp, #544]
-; CHECK-GI-NEXT: mov v1.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #672]
+; CHECK-GI-NEXT: ldr w8, [sp, #408]
; CHECK-GI-NEXT: mov v2.b[10], w11
+; CHECK-GI-NEXT: mov v4.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #664]
; CHECK-GI-NEXT: ldr w11, [sp, #40]
-; CHECK-GI-NEXT: mov v3.b[10], w9
+; CHECK-GI-NEXT: mov v5.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #160]
+; CHECK-GI-NEXT: mov v6.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #288]
+; CHECK-GI-NEXT: mov v7.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #544]
+; CHECK-GI-NEXT: mov v3.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #416]
+; CHECK-GI-NEXT: mov v2.b[11], w11
; CHECK-GI-NEXT: mov v4.b[10], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #168]
+; CHECK-GI-NEXT: ldr w10, [sp, #672]
+; CHECK-GI-NEXT: ldr w11, [sp, #48]
; CHECK-GI-NEXT: mov v5.b[10], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #296]
-; CHECK-GI-NEXT: mov v0.b[11], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #424]
-; CHECK-GI-NEXT: ldr w11, [sp, #552]
-; CHECK-GI-NEXT: mov v1.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #680]
-; CHECK-GI-NEXT: mov v2.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #48]
+; CHECK-GI-NEXT: ldr w8, [sp, #168]
+; CHECK-GI-NEXT: mov v6.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #296]
+; CHECK-GI-NEXT: mov v7.b[10], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #552]
; CHECK-GI-NEXT: mov v3.b[11], w8
-; CHECK-GI-NEXT: mov v4.b[11], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #176]
-; CHECK-GI-NEXT: mov v5.b[11], w9
-; CHECK-GI-NEXT: ldr w11, [sp, #304]
-; CHECK-GI-NEXT: mov v0.b[12], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #432]
-; CHECK-GI-NEXT: ldr w10, [sp, #560]
-; CHECK-GI-NEXT: mov v1.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #688]
+; CHECK-GI-NEXT: ldr w8, [sp, #424]
; CHECK-GI-NEXT: mov v2.b[12], w11
+; CHECK-GI-NEXT: mov v4.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #680]
; CHECK-GI-NEXT: ldr w11, [sp, #56]
-; CHECK-GI-NEXT: mov v3.b[12], w9
+; CHECK-GI-NEXT: mov v5.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #176]
+; CHECK-GI-NEXT: mov v6.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #304]
+; CHECK-GI-NEXT: mov v7.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #560]
+; CHECK-GI-NEXT: mov v3.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #432]
+; CHECK-GI-NEXT: mov v2.b[13], w11
; CHECK-GI-NEXT: mov v4.b[12], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #184]
+; CHECK-GI-NEXT: ldr w10, [sp, #688]
+; CHECK-GI-NEXT: ldr w11, [sp, #64]
; CHECK-GI-NEXT: mov v5.b[12], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #312]
-; CHECK-GI-NEXT: mov v0.b[13], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #440]
-; CHECK-GI-NEXT: ldr w11, [sp, #568]
-; CHECK-GI-NEXT: mov v1.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #696]
-; CHECK-GI-NEXT: mov v2.b[13], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #64]
+; CHECK-GI-NEXT: ldr w8, [sp, #184]
+; CHECK-GI-NEXT: mov v6.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #312]
+; CHECK-GI-NEXT: mov v7.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #568]
; CHECK-GI-NEXT: mov v3.b[13], w8
-; CHECK-GI-NEXT: mov v4.b[13], w11
-; CHECK-GI-NEXT: ldr w8, [sp, #192]
-; CHECK-GI-NEXT: mov v5.b[13], w9
-; CHECK-GI-NEXT: ldr w11, [sp, #320]
-; CHECK-GI-NEXT: mov v0.b[14], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #448]
-; CHECK-GI-NEXT: ldr w10, [sp, #576]
-; CHECK-GI-NEXT: mov v1.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #704]
+; CHECK-GI-NEXT: ldr w8, [sp, #440]
; CHECK-GI-NEXT: mov v2.b[14], w11
+; CHECK-GI-NEXT: mov v4.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #696]
; CHECK-GI-NEXT: ldr w11, [sp, #72]
-; CHECK-GI-NEXT: mov v3.b[14], w9
+; CHECK-GI-NEXT: mov v5.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #192]
+; CHECK-GI-NEXT: mov v6.b[13], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #320]
+; CHECK-GI-NEXT: mov v7.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #576]
+; CHECK-GI-NEXT: mov v3.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #448]
+; CHECK-GI-NEXT: mov v2.b[15], w11
; CHECK-GI-NEXT: mov v4.b[14], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #200]
-; CHECK-GI-NEXT: mov v5.b[14], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #328]
-; CHECK-GI-NEXT: mov v0.b[15], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #584]
-; CHECK-GI-NEXT: mov v1.b[15], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #712]
-; CHECK-GI-NEXT: mov v2.b[15], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #720]
-; CHECK-GI-NEXT: ldr w8, [sp, #456]
-; CHECK-GI-NEXT: mov v4.b[15], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #976]
-; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v5.b[15], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #856]
-; CHECK-GI-NEXT: fmov s6, w10
-; CHECK-GI-NEXT: fmov s16, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1104]
-; CHECK-GI-NEXT: ldr w10, [sp, #984]
-; CHECK-GI-NEXT: mov v7.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1360]
+; CHECK-GI-NEXT: ldr w10, [sp, #704]
+; CHECK-GI-NEXT: ldr w11, [sp, #848]
+; CHECK-GI-NEXT: mov v5.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #200]
+; CHECK-GI-NEXT: mov v6.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #328]
+; CHECK-GI-NEXT: mov v7.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #584]
; CHECK-GI-NEXT: mov v3.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #456]
; CHECK-GI-NEXT: fmov s17, w11
-; CHECK-GI-NEXT: mov v6.b[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1112]
-; CHECK-GI-NEXT: fmov s19, w9
+; CHECK-GI-NEXT: mov v4.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #712]
+; CHECK-GI-NEXT: ldr w11, [sp, #1104]
+; CHECK-GI-NEXT: mov v5.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #856]
+; CHECK-GI-NEXT: mov v6.b[15], w10
+; CHECK-GI-NEXT: mov v7.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #976]
+; CHECK-GI-NEXT: ldr w10, [sp, #728]
+; CHECK-GI-NEXT: mov v17.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1232]
+; CHECK-GI-NEXT: fmov s19, w11
+; CHECK-GI-NEXT: fmov s18, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1360]
; CHECK-GI-NEXT: mov v16.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1240]
-; CHECK-GI-NEXT: ldr w11, [sp, #1368]
-; CHECK-GI-NEXT: ldr w8, [sp, #736]
-; CHECK-GI-NEXT: ldr w9, [sp, #864]
-; CHECK-GI-NEXT: mov v17.b[1], w13
+; CHECK-GI-NEXT: ldr w10, [sp, #984]
+; CHECK-GI-NEXT: fmov s20, w8
+; CHECK-GI-NEXT: ldr w11, [sp, #1240]
+; CHECK-GI-NEXT: fmov s21, w9
+; CHECK-GI-NEXT: ldr w8, [sp, #1368]
+; CHECK-GI-NEXT: mov v19.b[1], w12
; CHECK-GI-NEXT: mov v18.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #992]
-; CHECK-GI-NEXT: mov v19.b[1], w11
-; CHECK-GI-NEXT: mov v6.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1120]
-; CHECK-GI-NEXT: mov v7.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1248]
-; CHECK-GI-NEXT: ldr w11, [sp, #1376]
+; CHECK-GI-NEXT: ldr w10, [sp, #736]
+; CHECK-GI-NEXT: ldr w9, [sp, #992]
+; CHECK-GI-NEXT: mov v20.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #864]
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v21.b[1], w8
; CHECK-GI-NEXT: mov v16.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #752]
-; CHECK-GI-NEXT: mov v20.s[2], wzr
-; CHECK-GI-NEXT: mov v17.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #872]
+; CHECK-GI-NEXT: ldr w8, [sp, #1120]
+; CHECK-GI-NEXT: mov v17.b[2], w11
+; CHECK-GI-NEXT: ldr w10, [sp, #1248]
+; CHECK-GI-NEXT: ldr w11, [sp, #744]
; CHECK-GI-NEXT: mov v18.b[2], w9
-; CHECK-GI-NEXT: mov v19.b[2], w11
-; CHECK-GI-NEXT: ldr w9, [sp, #1000]
-; CHECK-GI-NEXT: mov v6.b[3], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #1128]
-; CHECK-GI-NEXT: mov v7.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1256]
-; CHECK-GI-NEXT: ldr w12, [sp, #1384]
-; CHECK-GI-NEXT: mov v16.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #760]
-; CHECK-GI-NEXT: mov v17.b[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #880]
-; CHECK-GI-NEXT: mov v18.b[3], w8
-; CHECK-GI-NEXT: mov v19.b[3], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #1008]
-; CHECK-GI-NEXT: mov v6.b[4], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1136]
-; CHECK-GI-NEXT: mov v7.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1264]
-; CHECK-GI-NEXT: ldr w12, [sp, #1392]
-; CHECK-GI-NEXT: mov v16.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #768]
-; CHECK-GI-NEXT: mov v17.b[4], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #888]
-; CHECK-GI-NEXT: mov v18.b[4], w11
-; CHECK-GI-NEXT: mov v19.b[4], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #1016]
-; CHECK-GI-NEXT: mov v6.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1144]
-; CHECK-GI-NEXT: mov v7.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1272]
-; CHECK-GI-NEXT: ldr w12, [sp, #1400]
+; CHECK-GI-NEXT: ldr w9, [sp, #1376]
+; CHECK-GI-NEXT: mov v19.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #872]
+; CHECK-GI-NEXT: mov v20.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1000]
+; CHECK-GI-NEXT: mov v21.b[2], w9
+; CHECK-GI-NEXT: mov v16.b[3], w11
+; CHECK-GI-NEXT: ldr w9, [sp, #1256]
+; CHECK-GI-NEXT: mov v17.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1128]
+; CHECK-GI-NEXT: ldr w11, [sp, #752]
+; CHECK-GI-NEXT: mov v18.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1384]
+; CHECK-GI-NEXT: sdot v22.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT: mov v19.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #880]
+; CHECK-GI-NEXT: mov v20.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1008]
+; CHECK-GI-NEXT: mov v21.b[3], w10
+; CHECK-GI-NEXT: mov v16.b[4], w11
+; CHECK-GI-NEXT: mov v17.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1136]
+; CHECK-GI-NEXT: ldr w10, [sp, #1264]
+; CHECK-GI-NEXT: mov v18.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1392]
+; CHECK-GI-NEXT: ldr w11, [sp, #760]
+; CHECK-GI-NEXT: mov v19.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #888]
+; CHECK-GI-NEXT: mov v20.b[4], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1016]
+; CHECK-GI-NEXT: mov v21.b[4], w9
; CHECK-GI-NEXT: mov v16.b[5], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #776]
-; CHECK-GI-NEXT: mov v17.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #896]
+; CHECK-GI-NEXT: mov v17.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1144]
+; CHECK-GI-NEXT: ldr w9, [sp, #1272]
; CHECK-GI-NEXT: mov v18.b[5], w10
-; CHECK-GI-NEXT: mov v19.b[5], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #1024]
-; CHECK-GI-NEXT: mov v6.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1152]
-; CHECK-GI-NEXT: mov v7.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1280]
-; CHECK-GI-NEXT: ldr w12, [sp, #1408]
-; CHECK-GI-NEXT: mov v16.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #784]
+; CHECK-GI-NEXT: ldr w10, [sp, #1400]
+; CHECK-GI-NEXT: ldr w11, [sp, #768]
+; CHECK-GI-NEXT: mov v19.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #896]
+; CHECK-GI-NEXT: mov v20.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1024]
+; CHECK-GI-NEXT: mov v21.b[5], w10
+; CHECK-GI-NEXT: mov v16.b[6], w11
; CHECK-GI-NEXT: mov v17.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #904]
+; CHECK-GI-NEXT: ldr w8, [sp, #1152]
+; CHECK-GI-NEXT: ldr w10, [sp, #1280]
; CHECK-GI-NEXT: mov v18.b[6], w9
-; CHECK-GI-NEXT: mov v19.b[6], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #1032]
-; CHECK-GI-NEXT: mov v6.b[7], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1160]
-; CHECK-GI-NEXT: mov v7.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1288]
-; CHECK-GI-NEXT: ldr w12, [sp, #1416]
-; CHECK-GI-NEXT: mov v16.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #792]
-; CHECK-GI-NEXT: mov v17.b[7], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #912]
-; CHECK-GI-NEXT: mov v18.b[7], w8
-; CHECK-GI-NEXT: mov v19.b[7], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #1040]
-; CHECK-GI-NEXT: mov v6.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1168]
-; CHECK-GI-NEXT: mov v7.b[8], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1296]
-; CHECK-GI-NEXT: ldr w12, [sp, #1424]
-; CHECK-GI-NEXT: mov v16.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #800]
-; CHECK-GI-NEXT: mov v17.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #920]
-; CHECK-GI-NEXT: mov v18.b[8], w11
-; CHECK-GI-NEXT: mov v19.b[8], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #1048]
-; CHECK-GI-NEXT: mov v6.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1176]
-; CHECK-GI-NEXT: mov v7.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1304]
-; CHECK-GI-NEXT: ldr w12, [sp, #1432]
+; CHECK-GI-NEXT: ldr w9, [sp, #1408]
+; CHECK-GI-NEXT: ldr w11, [sp, #776]
+; CHECK-GI-NEXT: mov v19.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #904]
+; CHECK-GI-NEXT: mov v20.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1032]
+; CHECK-GI-NEXT: mov v21.b[6], w9
+; CHECK-GI-NEXT: mov v16.b[7], w11
+; CHECK-GI-NEXT: mov v17.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1160]
+; CHECK-GI-NEXT: ldr w9, [sp, #1288]
+; CHECK-GI-NEXT: mov v18.b[7], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1416]
+; CHECK-GI-NEXT: ldr w11, [sp, #784]
+; CHECK-GI-NEXT: mov v19.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #912]
+; CHECK-GI-NEXT: mov v20.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1040]
+; CHECK-GI-NEXT: mov v21.b[7], w10
+; CHECK-GI-NEXT: mov v16.b[8], w11
+; CHECK-GI-NEXT: mov v17.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1168]
+; CHECK-GI-NEXT: ldr w10, [sp, #1296]
+; CHECK-GI-NEXT: mov v18.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1424]
+; CHECK-GI-NEXT: ldr w11, [sp, #792]
+; CHECK-GI-NEXT: mov v19.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #920]
+; CHECK-GI-NEXT: mov v20.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1048]
+; CHECK-GI-NEXT: mov v21.b[8], w9
; CHECK-GI-NEXT: mov v16.b[9], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #808]
-; CHECK-GI-NEXT: mov v17.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #928]
+; CHECK-GI-NEXT: mov v17.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1176]
+; CHECK-GI-NEXT: ldr w9, [sp, #1304]
; CHECK-GI-NEXT: mov v18.b[9], w10
-; CHECK-GI-NEXT: mov v19.b[9], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #1056]
-; CHECK-GI-NEXT: mov v6.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1184]
-; CHECK-GI-NEXT: mov v7.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1312]
-; CHECK-GI-NEXT: ldr w12, [sp, #1440]
-; CHECK-GI-NEXT: mov v16.b[10], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #816]
+; CHECK-GI-NEXT: ldr w10, [sp, #1432]
+; CHECK-GI-NEXT: ldr w11, [sp, #800]
+; CHECK-GI-NEXT: mov v19.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #928]
+; CHECK-GI-NEXT: mov v20.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1056]
+; CHECK-GI-NEXT: mov v21.b[9], w10
+; CHECK-GI-NEXT: mov v16.b[10], w11
; CHECK-GI-NEXT: mov v17.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #936]
+; CHECK-GI-NEXT: ldr w8, [sp, #1184]
+; CHECK-GI-NEXT: ldr w10, [sp, #1312]
; CHECK-GI-NEXT: mov v18.b[10], w9
-; CHECK-GI-NEXT: mov v19.b[10], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #1064]
-; CHECK-GI-NEXT: mov v6.b[11], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1192]
-; CHECK-GI-NEXT: mov v7.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1320]
-; CHECK-GI-NEXT: ldr w12, [sp, #1448]
-; CHECK-GI-NEXT: mov v16.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #824]
-; CHECK-GI-NEXT: mov v17.b[11], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #944]
-; CHECK-GI-NEXT: mov v18.b[11], w8
-; CHECK-GI-NEXT: mov v19.b[11], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #1072]
-; CHECK-GI-NEXT: mov v6.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1200]
-; CHECK-GI-NEXT: mov v7.b[12], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1328]
-; CHECK-GI-NEXT: ldr w12, [sp, #1456]
-; CHECK-GI-NEXT: mov v16.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #832]
-; CHECK-GI-NEXT: mov v17.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #952]
-; CHECK-GI-NEXT: mov v18.b[12], w11
-; CHECK-GI-NEXT: mov v19.b[12], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #1080]
-; CHECK-GI-NEXT: mov v6.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1208]
-; CHECK-GI-NEXT: mov v7.b[13], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1336]
-; CHECK-GI-NEXT: ldr w12, [sp, #1464]
+; CHECK-GI-NEXT: ldr w9, [sp, #1440]
+; CHECK-GI-NEXT: ldr w11, [sp, #808]
+; CHECK-GI-NEXT: mov v19.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #936]
+; CHECK-GI-NEXT: mov v20.b[10], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1064]
+; CHECK-GI-NEXT: mov v21.b[10], w9
+; CHECK-GI-NEXT: mov v16.b[11], w11
+; CHECK-GI-NEXT: mov v17.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1192]
+; CHECK-GI-NEXT: ldr w9, [sp, #1320]
+; CHECK-GI-NEXT: mov v18.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1448]
+; CHECK-GI-NEXT: ldr w11, [sp, #816]
+; CHECK-GI-NEXT: mov v19.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #944]
+; CHECK-GI-NEXT: mov v20.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1072]
+; CHECK-GI-NEXT: mov v21.b[11], w10
+; CHECK-GI-NEXT: mov v16.b[12], w11
+; CHECK-GI-NEXT: mov v17.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1200]
+; CHECK-GI-NEXT: ldr w10, [sp, #1328]
+; CHECK-GI-NEXT: mov v18.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1456]
+; CHECK-GI-NEXT: ldr w11, [sp, #824]
+; CHECK-GI-NEXT: mov v19.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #952]
+; CHECK-GI-NEXT: mov v20.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1080]
+; CHECK-GI-NEXT: mov v21.b[12], w9
; CHECK-GI-NEXT: mov v16.b[13], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #960]
-; CHECK-GI-NEXT: mov v17.b[13], w9
+; CHECK-GI-NEXT: mov v17.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1208]
+; CHECK-GI-NEXT: ldr w9, [sp, #1336]
; CHECK-GI-NEXT: mov v18.b[13], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1464]
+; CHECK-GI-NEXT: ldr w11, [sp, #832]
+; CHECK-GI-NEXT: mov v19.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #960]
+; CHECK-GI-NEXT: mov v20.b[13], w9
; CHECK-GI-NEXT: ldr w9, [sp, #1088]
-; CHECK-GI-NEXT: mov v19.b[13], w12
-; CHECK-GI-NEXT: mov v6.b[14], w8
+; CHECK-GI-NEXT: mov v21.b[13], w10
+; CHECK-GI-NEXT: mov v16.b[14], w11
+; CHECK-GI-NEXT: mov v17.b[14], w8
; CHECK-GI-NEXT: ldr w8, [sp, #1216]
-; CHECK-GI-NEXT: mov v7.b[14], w11
; CHECK-GI-NEXT: ldr w10, [sp, #1344]
-; CHECK-GI-NEXT: ldr w11, [sp, #1472]
-; CHECK-GI-NEXT: ldr w12, [sp, #840]
-; CHECK-GI-NEXT: mov v16.b[14], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1096]
-; CHECK-GI-NEXT: mov v17.b[14], w8
-; CHECK-GI-NEXT: mov v18.b[14], w10
+; CHECK-GI-NEXT: mov v18.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1472]
+; CHECK-GI-NEXT: ldr w11, [sp, #840]
+; CHECK-GI-NEXT: mov v19.b[14], w8
; CHECK-GI-NEXT: ldr w8, [sp, #968]
-; CHECK-GI-NEXT: mov v19.b[14], w11
-; CHECK-GI-NEXT: ldr w10, [sp, #1224]
-; CHECK-GI-NEXT: mov v6.b[15], w12
+; CHECK-GI-NEXT: mov v20.b[14], w10
+; CHECK-GI-NEXT: mov v21.b[14], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #1096]
+; CHECK-GI-NEXT: mov v16.b[15], w11
+; CHECK-GI-NEXT: ldr w9, [sp, #1224]
; CHECK-GI-NEXT: ldr w11, [sp, #1352]
-; CHECK-GI-NEXT: ldr w12, [sp, #1480]
-; CHECK-GI-NEXT: mov v7.b[15], w8
-; CHECK-GI-NEXT: mov v16.b[15], w9
-; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v17.b[15], w10
-; CHECK-GI-NEXT: mov v18.b[15], w11
-; CHECK-GI-NEXT: movi v26.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v19.b[15], w12
-; CHECK-GI-NEXT: sdot v21.4s, v0.16b, v3.16b
-; CHECK-GI-NEXT: sdot v22.4s, v1.16b, v4.16b
-; CHECK-GI-NEXT: sdot v23.4s, v2.16b, v5.16b
-; CHECK-GI-NEXT: mov v20.s[3], wzr
-; CHECK-GI-NEXT: sdot v25.4s, v6.16b, v17.16b
-; CHECK-GI-NEXT: sdot v26.4s, v7.16b, v18.16b
-; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
-; CHECK-GI-NEXT: add v0.4s, v21.4s, v22.4s
-; CHECK-GI-NEXT: add v1.4s, v23.4s, v20.4s
-; CHECK-GI-NEXT: add v2.4s, v25.4s, v26.4s
-; CHECK-GI-NEXT: add v3.4s, v24.4s, v20.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: mov v17.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1480]
+; CHECK-GI-NEXT: mov v18.b[15], w10
+; CHECK-GI-NEXT: sdot v23.4s, v3.16b, v6.16b
+; CHECK-GI-NEXT: mov v19.b[15], w9
+; CHECK-GI-NEXT: mov v20.b[15], w11
+; CHECK-GI-NEXT: sdot v24.4s, v4.16b, v7.16b
+; CHECK-GI-NEXT: mov v21.b[15], w8
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: add v2.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: sdot v25.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT: sdot v27.4s, v17.16b, v20.16b
+; CHECK-GI-NEXT: sdot v26.4s, v18.16b, v21.16b
+; CHECK-GI-NEXT: add v1.4s, v24.4s, v1.4s
+; CHECK-GI-NEXT: add v3.4s, v25.4s, v27.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v26.4s, v0.4s
; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
@@ -6746,217 +6966,221 @@ define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w10, [sp, #80]
-; CHECK-GI-NEXT: ldr w9, [sp, #88]
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w11, [sp, #208]
-; CHECK-GI-NEXT: ldr w8, [sp, #216]
-; CHECK-GI-NEXT: mov v6.s[0], wzr
-; CHECK-GI-NEXT: fmov s1, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #720]
-; CHECK-GI-NEXT: movi v7.16b, #1
-; CHECK-GI-NEXT: fmov s2, w11
-; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: ldr w11, [sp, #728]
-; CHECK-GI-NEXT: fmov s3, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #976]
-; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #848]
-; CHECK-GI-NEXT: mov v6.s[1], wzr
-; CHECK-GI-NEXT: fmov s5, w10
-; CHECK-GI-NEXT: mov v2.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #856]
+; CHECK-GI-NEXT: ldr w9, [sp, #208]
+; CHECK-GI-NEXT: ldr w11, [sp, #80]
+; CHECK-GI-NEXT: fmov s2, w0
+; CHECK-GI-NEXT: ldr w10, [sp, #216]
+; CHECK-GI-NEXT: ldr w12, [sp, #848]
+; CHECK-GI-NEXT: fmov s1, wzr
; CHECK-GI-NEXT: fmov s4, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #984]
-; CHECK-GI-NEXT: mov v3.b[1], w11
-; CHECK-GI-NEXT: ldr w10, [sp, #224]
-; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: ldr w11, [sp, #16]
-; CHECK-GI-NEXT: mov v5.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #864]
-; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v4.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: mov v2.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #992]
-; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #736]
-; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: mov v5.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #872]
-; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v3.b[2], w8
-; CHECK-GI-NEXT: mov v4.b[2], w9
+; CHECK-GI-NEXT: fmov s3, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #720]
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: fmov s6, w12
+; CHECK-GI-NEXT: ldr w9, [sp, #856]
+; CHECK-GI-NEXT: fmov s5, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #984]
+; CHECK-GI-NEXT: mov v2.b[1], w1
+; CHECK-GI-NEXT: mov v4.b[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #976]
+; CHECK-GI-NEXT: mov v3.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #728]
+; CHECK-GI-NEXT: mov v6.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #736]
+; CHECK-GI-NEXT: fmov s7, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #96]
+; CHECK-GI-NEXT: ldr w12, [sp, #16]
+; CHECK-GI-NEXT: mov v5.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #224]
+; CHECK-GI-NEXT: mov v2.b[2], w2
+; CHECK-GI-NEXT: mov v3.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #864]
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: mov v7.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: mov v4.b[2], w8
; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: mov v6.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #872]
+; CHECK-GI-NEXT: mov v5.b[2], w9
; CHECK-GI-NEXT: ldr w9, [sp, #232]
-; CHECK-GI-NEXT: mov v6.s[2], wzr
-; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #744]
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: mov v2.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1000]
+; CHECK-GI-NEXT: mov v2.b[3], w3
; CHECK-GI-NEXT: mov v3.b[3], w8
-; CHECK-GI-NEXT: mov v4.b[3], w10
-; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: ldr w10, [sp, #240]
-; CHECK-GI-NEXT: mov v5.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #880]
-; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #752]
-; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v2.b[4], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1008]
-; CHECK-GI-NEXT: mov v6.s[3], wzr
-; CHECK-GI-NEXT: mov v3.b[4], w8
-; CHECK-GI-NEXT: mov v4.b[4], w9
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: mov v7.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1000]
+; CHECK-GI-NEXT: mov v4.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #112]
+; CHECK-GI-NEXT: mov v6.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #880]
+; CHECK-GI-NEXT: mov v5.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #240]
+; CHECK-GI-NEXT: mov v2.b[4], w4
+; CHECK-GI-NEXT: mov v3.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #752]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v7.b[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1008]
+; CHECK-GI-NEXT: mov v4.b[4], w8
; CHECK-GI-NEXT: ldr w8, [sp, #120]
-; CHECK-GI-NEXT: ldr w9, [sp, #248]
-; CHECK-GI-NEXT: mov v5.b[4], w10
+; CHECK-GI-NEXT: mov v6.b[4], w10
; CHECK-GI-NEXT: ldr w10, [sp, #888]
-; CHECK-GI-NEXT: mov v1.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #760]
-; CHECK-GI-NEXT: mov v0.b[6], w6
-; CHECK-GI-NEXT: mov v2.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1016]
+; CHECK-GI-NEXT: mov v5.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #248]
+; CHECK-GI-NEXT: mov v2.b[5], w5
; CHECK-GI-NEXT: mov v3.b[5], w8
-; CHECK-GI-NEXT: mov v4.b[5], w10
-; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: ldr w10, [sp, #256]
-; CHECK-GI-NEXT: mov v5.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #896]
-; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #768]
-; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v2.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1024]
-; CHECK-GI-NEXT: mov v3.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #760]
+; CHECK-GI-NEXT: movi v16.16b, #1
+; CHECK-GI-NEXT: mov v7.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1016]
+; CHECK-GI-NEXT: mov v4.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #128]
+; CHECK-GI-NEXT: mov v6.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #896]
+; CHECK-GI-NEXT: mov v5.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #256]
+; CHECK-GI-NEXT: mov v2.b[6], w6
+; CHECK-GI-NEXT: mov v3.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #768]
+; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v7.b[5], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1024]
+; CHECK-GI-NEXT: mov v4.b[6], w8
; CHECK-GI-NEXT: ldr w8, [sp, #136]
-; CHECK-GI-NEXT: mov v4.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #264]
-; CHECK-GI-NEXT: mov v5.b[6], w10
+; CHECK-GI-NEXT: mov v6.b[6], w10
; CHECK-GI-NEXT: ldr w10, [sp, #904]
-; CHECK-GI-NEXT: mov v1.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #776]
-; CHECK-GI-NEXT: mov v0.b[8], w11
-; CHECK-GI-NEXT: mov v2.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1032]
-; CHECK-GI-NEXT: ldr w11, [sp, #24]
+; CHECK-GI-NEXT: mov v5.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #264]
+; CHECK-GI-NEXT: mov v2.b[7], w7
; CHECK-GI-NEXT: mov v3.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #144]
-; CHECK-GI-NEXT: mov v4.b[7], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #272]
-; CHECK-GI-NEXT: mov v5.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #912]
-; CHECK-GI-NEXT: mov v1.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #784]
-; CHECK-GI-NEXT: mov v0.b[9], w11
-; CHECK-GI-NEXT: mov v2.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1040]
-; CHECK-GI-NEXT: ldr w11, [sp, #32]
-; CHECK-GI-NEXT: mov v3.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #776]
+; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v7.b[6], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1032]
+; CHECK-GI-NEXT: mov v4.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #144]
+; CHECK-GI-NEXT: mov v6.b[7], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #912]
+; CHECK-GI-NEXT: mov v5.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #272]
+; CHECK-GI-NEXT: mov v2.b[8], w12
+; CHECK-GI-NEXT: mov v3.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #784]
+; CHECK-GI-NEXT: ldr w12, [sp, #24]
+; CHECK-GI-NEXT: mov v7.b[7], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1040]
+; CHECK-GI-NEXT: mov v4.b[8], w8
; CHECK-GI-NEXT: ldr w8, [sp, #152]
-; CHECK-GI-NEXT: mov v4.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #280]
-; CHECK-GI-NEXT: mov v5.b[8], w10
+; CHECK-GI-NEXT: mov v6.b[8], w10
; CHECK-GI-NEXT: ldr w10, [sp, #920]
-; CHECK-GI-NEXT: mov v1.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #792]
-; CHECK-GI-NEXT: mov v0.b[10], w11
-; CHECK-GI-NEXT: mov v2.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1048]
-; CHECK-GI-NEXT: ldr w11, [sp, #40]
+; CHECK-GI-NEXT: mov v5.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #280]
+; CHECK-GI-NEXT: mov v2.b[9], w12
; CHECK-GI-NEXT: mov v3.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: mov v4.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #288]
-; CHECK-GI-NEXT: mov v5.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #928]
-; CHECK-GI-NEXT: mov v1.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #800]
-; CHECK-GI-NEXT: mov v0.b[11], w11
-; CHECK-GI-NEXT: mov v2.b[10], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1056]
-; CHECK-GI-NEXT: ldr w11, [sp, #48]
-; CHECK-GI-NEXT: mov v3.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #792]
+; CHECK-GI-NEXT: ldr w12, [sp, #32]
+; CHECK-GI-NEXT: mov v7.b[8], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1048]
+; CHECK-GI-NEXT: mov v4.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #160]
+; CHECK-GI-NEXT: mov v6.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #928]
+; CHECK-GI-NEXT: mov v5.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #288]
+; CHECK-GI-NEXT: mov v2.b[10], w12
+; CHECK-GI-NEXT: mov v3.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #800]
+; CHECK-GI-NEXT: ldr w12, [sp, #40]
+; CHECK-GI-NEXT: mov v7.b[9], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1056]
+; CHECK-GI-NEXT: mov v4.b[10], w8
; CHECK-GI-NEXT: ldr w8, [sp, #168]
-; CHECK-GI-NEXT: mov v4.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #296]
-; CHECK-GI-NEXT: mov v5.b[10], w10
+; CHECK-GI-NEXT: mov v6.b[10], w10
; CHECK-GI-NEXT: ldr w10, [sp, #936]
-; CHECK-GI-NEXT: mov v1.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #808]
-; CHECK-GI-NEXT: mov v0.b[12], w11
-; CHECK-GI-NEXT: mov v2.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1064]
-; CHECK-GI-NEXT: ldr w11, [sp, #56]
+; CHECK-GI-NEXT: mov v5.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #296]
+; CHECK-GI-NEXT: mov v2.b[11], w12
; CHECK-GI-NEXT: mov v3.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #176]
-; CHECK-GI-NEXT: mov v4.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #304]
-; CHECK-GI-NEXT: mov v5.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #944]
-; CHECK-GI-NEXT: mov v1.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #816]
-; CHECK-GI-NEXT: mov v0.b[13], w11
-; CHECK-GI-NEXT: mov v2.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1072]
-; CHECK-GI-NEXT: ldr w11, [sp, #64]
-; CHECK-GI-NEXT: mov v3.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #808]
+; CHECK-GI-NEXT: ldr w12, [sp, #48]
+; CHECK-GI-NEXT: mov v7.b[10], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1064]
+; CHECK-GI-NEXT: mov v4.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #176]
+; CHECK-GI-NEXT: mov v6.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #944]
+; CHECK-GI-NEXT: mov v5.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #304]
+; CHECK-GI-NEXT: mov v2.b[12], w12
+; CHECK-GI-NEXT: mov v3.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #816]
+; CHECK-GI-NEXT: ldr w12, [sp, #56]
+; CHECK-GI-NEXT: mov v7.b[11], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1072]
+; CHECK-GI-NEXT: mov v4.b[12], w8
; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: mov v4.b[12], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #312]
-; CHECK-GI-NEXT: mov v5.b[12], w10
+; CHECK-GI-NEXT: mov v6.b[12], w10
; CHECK-GI-NEXT: ldr w10, [sp, #952]
-; CHECK-GI-NEXT: mov v1.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #824]
-; CHECK-GI-NEXT: mov v0.b[14], w11
-; CHECK-GI-NEXT: mov v2.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1080]
-; CHECK-GI-NEXT: ldr w11, [sp, #72]
+; CHECK-GI-NEXT: mov v5.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #312]
+; CHECK-GI-NEXT: mov v2.b[13], w12
; CHECK-GI-NEXT: mov v3.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #192]
-; CHECK-GI-NEXT: mov v4.b[13], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #320]
-; CHECK-GI-NEXT: mov v5.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #960]
-; CHECK-GI-NEXT: mov v1.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #832]
-; CHECK-GI-NEXT: mov v0.b[15], w11
-; CHECK-GI-NEXT: mov v2.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1088]
-; CHECK-GI-NEXT: ldr w11, [sp, #968]
-; CHECK-GI-NEXT: mov v3.b[14], w8
-; CHECK-GI-NEXT: mov v4.b[14], w9
+; CHECK-GI-NEXT: ldr w8, [sp, #824]
+; CHECK-GI-NEXT: ldr w12, [sp, #64]
+; CHECK-GI-NEXT: mov v7.b[12], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1080]
+; CHECK-GI-NEXT: mov v4.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #192]
+; CHECK-GI-NEXT: mov v6.b[13], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #960]
+; CHECK-GI-NEXT: mov v5.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #320]
+; CHECK-GI-NEXT: mov v2.b[14], w12
+; CHECK-GI-NEXT: mov v3.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #832]
+; CHECK-GI-NEXT: ldr w12, [sp, #72]
+; CHECK-GI-NEXT: mov v7.b[13], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1088]
+; CHECK-GI-NEXT: mov v4.b[14], w8
; CHECK-GI-NEXT: ldr w8, [sp, #200]
-; CHECK-GI-NEXT: mov v5.b[14], w10
+; CHECK-GI-NEXT: mov v6.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #968]
+; CHECK-GI-NEXT: mov v5.b[14], w9
; CHECK-GI-NEXT: ldr w9, [sp, #328]
-; CHECK-GI-NEXT: ldr w10, [sp, #840]
-; CHECK-GI-NEXT: mov v1.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1096]
-; CHECK-GI-NEXT: sdot v16.4s, v0.16b, v7.16b
-; CHECK-GI-NEXT: mov v2.b[15], w9
-; CHECK-GI-NEXT: mov v3.b[15], w10
-; CHECK-GI-NEXT: mov v4.b[15], w11
+; CHECK-GI-NEXT: mov v2.b[15], w12
+; CHECK-GI-NEXT: mov v3.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #840]
+; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v7.b[14], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1096]
+; CHECK-GI-NEXT: mov v4.b[15], w9
+; CHECK-GI-NEXT: mov v6.b[15], w10
+; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
; CHECK-GI-NEXT: mov v5.b[15], w8
-; CHECK-GI-NEXT: sdot v17.4s, v1.16b, v7.16b
-; CHECK-GI-NEXT: sdot v18.4s, v2.16b, v7.16b
-; CHECK-GI-NEXT: sdot v19.4s, v3.16b, v7.16b
-; CHECK-GI-NEXT: sdot v21.4s, v4.16b, v7.16b
-; CHECK-GI-NEXT: sdot v20.4s, v5.16b, v7.16b
-; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT: add v1.4s, v18.4s, v6.4s
-; CHECK-GI-NEXT: add v2.4s, v19.4s, v21.4s
-; CHECK-GI-NEXT: add v3.4s, v20.4s, v6.4s
-; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: sdot v17.4s, v2.16b, v16.16b
+; CHECK-GI-NEXT: sdot v18.4s, v3.16b, v16.16b
+; CHECK-GI-NEXT: mov v7.b[15], w11
+; CHECK-GI-NEXT: sdot v19.4s, v4.16b, v16.16b
+; CHECK-GI-NEXT: sdot v21.4s, v6.16b, v16.16b
+; CHECK-GI-NEXT: sdot v20.4s, v5.16b, v16.16b
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: add v2.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: sdot v22.4s, v7.16b, v16.16b
+; CHECK-GI-NEXT: add v1.4s, v19.4s, v1.4s
+; CHECK-GI-NEXT: add v3.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: add v0.4s, v22.4s, v0.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: fmov w9, s0
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/phi.ll b/llvm/test/CodeGen/AArch64/phi.ll
index 55942d0e421bb..02842c04bf7bf 100644
--- a/llvm/test/CodeGen/AArch64/phi.ll
+++ b/llvm/test/CodeGen/AArch64/phi.ll
@@ -317,7 +317,7 @@ define <2 x i8> @tv2i8(i1 %c, ptr %p, <2 x i8> %a, <2 x i8> %b) {
; CHECK-GI-NEXT: .LBB10_3: // %e
; CHECK-GI-NEXT: umov w8, v1.b[0]
; CHECK-GI-NEXT: umov w9, v1.b[1]
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
@@ -895,26 +895,27 @@ define <3 x ptr> @tv3p0(i1 %c, ptr %p, <3 x ptr> %a, <3 x ptr> %b) {
;
; CHECK-GI-LABEL: tv3p0:
; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT: // kill: def $d5 killed $d5 def $q5
; CHECK-GI-NEXT: tbz w0, #0, .LBB30_2
; CHECK-GI-NEXT: // %bb.1: // %t
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d2
-; CHECK-GI-NEXT: str wzr, [x1]
-; CHECK-GI-NEXT: mov v0.d[0], x8
; CHECK-GI-NEXT: fmov x8, d1
+; CHECK-GI-NEXT: mov v5.16b, v2.16b
+; CHECK-GI-NEXT: str wzr, [x1]
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: mov v3.16b, v0.16b
; CHECK-GI-NEXT: b .LBB30_3
; CHECK-GI-NEXT: .LBB30_2:
-; CHECK-GI-NEXT: fmov x8, d3
-; CHECK-GI-NEXT: fmov x9, d5
-; CHECK-GI-NEXT: mov v0.d[0], x8
; CHECK-GI-NEXT: fmov x8, d4
+; CHECK-GI-NEXT: mov v3.d[1], x8
; CHECK-GI-NEXT: .LBB30_3: // %e
-; CHECK-GI-NEXT: mov v1.d[0], x9
-; CHECK-GI-NEXT: mov v0.d[1], x8
-; CHECK-GI-NEXT: mov d2, v0.d[1]
-; CHECK-GI-NEXT: fmov x10, d1
-; CHECK-GI-NEXT: fmov d1, d2
-; CHECK-GI-NEXT: fmov d2, x10
+; CHECK-GI-NEXT: mov d0, v3.d[1]
+; CHECK-GI-NEXT: fmov d2, d5
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov d0, d3
+; CHECK-GI-NEXT: fmov d1, x9
; CHECK-GI-NEXT: ret
entry:
br i1 %c, label %t, label %e
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index 61f221988777f..c158d8ad93b05 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -652,11 +652,8 @@ define i32 @ctpop_into_extract(ptr %p) {
; CHECKO0-LABEL: ctpop_into_extract:
; CHECKO0: // %bb.0:
; CHECKO0-NEXT: mov w8, #-1 // =0xffffffff
-; CHECKO0-NEXT: // implicit-def: $d1
-; CHECKO0-NEXT: // implicit-def: $q0
-; CHECKO0-NEXT: fmov d0, d1
-; CHECKO0-NEXT: mov v0.s[0], w8
-; CHECKO0-NEXT: fmov d2, d0
+; CHECKO0-NEXT: // implicit-def: $d2
+; CHECKO0-NEXT: fmov s2, w8
; CHECKO0-NEXT: ldr d0, [x0]
; CHECKO0-NEXT: fmov s1, s0
; CHECKO0-NEXT: fmov w8, s1
@@ -709,12 +706,12 @@ define i32 @ctpop_into_extract(ptr %p) {
; GISEL-LABEL: ctpop_into_extract:
; GISEL: // %bb.0:
; GISEL-NEXT: ldr d0, [x0]
-; GISEL-NEXT: mov w9, #-1 // =0xffffffff
; GISEL-NEXT: mov x8, x0
-; GISEL-NEXT: mov v2.s[0], w9
; GISEL-NEXT: mov w0, wzr
-; GISEL-NEXT: fmov w10, s0
-; GISEL-NEXT: fmov s1, w10
+; GISEL-NEXT: fmov w9, s0
+; GISEL-NEXT: fmov s1, w9
+; GISEL-NEXT: mov w9, #-1 // =0xffffffff
+; GISEL-NEXT: fmov s2, w9
; GISEL-NEXT: cnt v1.8b, v1.8b
; GISEL-NEXT: uaddlv h1, v1.8b
; GISEL-NEXT: mov v2.s[1], v1.s[0]
@@ -725,11 +722,8 @@ define i32 @ctpop_into_extract(ptr %p) {
; GISELO0-LABEL: ctpop_into_extract:
; GISELO0: // %bb.0:
; GISELO0-NEXT: mov w8, #-1 // =0xffffffff
-; GISELO0-NEXT: // implicit-def: $d1
-; GISELO0-NEXT: // implicit-def: $q0
-; GISELO0-NEXT: fmov d0, d1
-; GISELO0-NEXT: mov v0.s[0], w8
-; GISELO0-NEXT: fmov d2, d0
+; GISELO0-NEXT: // implicit-def: $d2
+; GISELO0-NEXT: fmov s2, w8
; GISELO0-NEXT: ldr d0, [x0]
; GISELO0-NEXT: fmov s1, s0
; GISELO0-NEXT: fmov w8, s1
diff --git a/llvm/test/CodeGen/AArch64/ptradd.ll b/llvm/test/CodeGen/AArch64/ptradd.ll
index 427542377bfda..28a8f4303765b 100644
--- a/llvm/test/CodeGen/AArch64/ptradd.ll
+++ b/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -90,18 +90,17 @@ define <3 x ptr> @vector_gep_v3i32(<3 x ptr> %b, <3 x i32> %off) {
; CHECK-GI-LABEL: vector_gep_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: smov x9, v3.s[0]
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: smov x8, v3.s[1]
-; CHECK-GI-NEXT: mov v4.d[0], x9
-; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: mov v0.d[1], x9
-; CHECK-GI-NEXT: fmov x9, d2
-; CHECK-GI-NEXT: mov v4.d[1], x8
+; CHECK-GI-NEXT: smov x10, v3.s[1]
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: fmov x8, d1
+; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov w8, v3.s[2]
-; CHECK-GI-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-GI-NEXT: fmov d1, x9
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mov v1.d[1], x10
; CHECK-GI-NEXT: add x8, x9, w8, sxtw
; CHECK-GI-NEXT: fmov d2, x8
+; CHECK-GI-NEXT: add v0.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: ret
entry:
@@ -164,13 +163,12 @@ define <3 x ptr> @vector_gep_v3i64(<3 x ptr> %b, <3 x i64> %off) {
;
; CHECK-GI-LABEL: vector_gep_v3i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x8, d1
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT: // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT: fmov x9, d5
; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: fmov x8, d1
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: fmov x8, d2
; CHECK-GI-NEXT: add x8, x8, x9
@@ -201,19 +199,12 @@ entry:
}
define <2 x ptr> @vector_gep_v4i128(<2 x ptr> %b, <2 x i128> %off) {
-; CHECK-SD-LABEL: vector_gep_v4i128:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov d1, x0
-; CHECK-SD-NEXT: mov v1.d[1], x2
-; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vector_gep_v4i128:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v1.d[0], x0
-; CHECK-GI-NEXT: mov v1.d[1], x2
-; CHECK-GI-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vector_gep_v4i128:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov d1, x0
+; CHECK-NEXT: mov v1.d[1], x2
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ret
entry:
%g = getelementptr i8, <2 x ptr> %b, <2 x i128> %off
ret <2 x ptr> %g
diff --git a/llvm/test/CodeGen/AArch64/rem.ll b/llvm/test/CodeGen/AArch64/rem.ll
index d807635f5d87d..7477d33f9aa46 100644
--- a/llvm/test/CodeGen/AArch64/rem.ll
+++ b/llvm/test/CodeGen/AArch64/rem.ll
@@ -190,7 +190,7 @@ define <2 x i8> @sv2i8(<2 x i8> %d, <2 x i8> %e) {
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -284,7 +284,7 @@ define <4 x i8> @sv4i8(<4 x i8> %d, <4 x i8> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -363,12 +363,12 @@ define <8 x i8> @sv8i8(<8 x i8> %d, <8 x i8> %e) {
; CHECK-GI-NEXT: fmov w13, s1
; CHECK-GI-NEXT: mov w14, v1.s[1]
; CHECK-GI-NEXT: mov w15, v1.s[2]
+; CHECK-GI-NEXT: mov w16, v1.s[3]
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v2.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v0.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -381,11 +381,11 @@ define <8 x i8> @sv8i8(<8 x i8> %d, <8 x i8> %e) {
; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
; CHECK-GI-NEXT: sdiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: sdiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v0.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: sdiv w8, w8, w12
+; CHECK-GI-NEXT: sdiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
@@ -527,20 +527,20 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: mov w18, v1.s[1]
; CHECK-GI-NEXT: mov w0, v1.s[2]
; CHECK-GI-NEXT: mov w1, v1.s[3]
-; CHECK-GI-NEXT: sdiv w11, w8, w9
+; CHECK-GI-NEXT: sdiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[1]
; CHECK-GI-NEXT: mov w9, v3.s[1]
; CHECK-GI-NEXT: fmov w2, s7
; CHECK-GI-NEXT: mov w3, v7.s[1]
; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: sdiv w10, w8, w9
+; CHECK-GI-NEXT: mov w5, v7.s[3]
+; CHECK-GI-NEXT: sdiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[2]
; CHECK-GI-NEXT: mov w9, v3.s[2]
-; CHECK-GI-NEXT: mov v16.s[0], w11
-; CHECK-GI-NEXT: mov w11, v6.s[3]
+; CHECK-GI-NEXT: fmov s16, w10
; CHECK-GI-NEXT: sdiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[3]
-; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: mov v16.s[1], w11
; CHECK-GI-NEXT: sdiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
; CHECK-GI-NEXT: mov v16.s[2], w9
@@ -552,8 +552,7 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: sdiv w15, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[2]
; CHECK-GI-NEXT: mov w13, v5.s[2]
-; CHECK-GI-NEXT: mov v17.s[0], w14
-; CHECK-GI-NEXT: mov w14, v7.s[3]
+; CHECK-GI-NEXT: fmov s17, w14
; CHECK-GI-NEXT: sdiv w13, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: mov v17.s[1], w15
@@ -566,7 +565,7 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
; CHECK-GI-NEXT: sdiv w17, w17, w18
; CHECK-GI-NEXT: mov w18, v0.s[2]
-; CHECK-GI-NEXT: mov v18.s[0], w16
+; CHECK-GI-NEXT: fmov s18, w16
; CHECK-GI-NEXT: sdiv w18, w18, w0
; CHECK-GI-NEXT: mov w0, v0.s[3]
; CHECK-GI-NEXT: mov v18.s[1], w17
@@ -580,10 +579,11 @@ define <16 x i8> @sv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
; CHECK-GI-NEXT: sdiv w2, w2, w3
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: mov v19.s[0], w1
+; CHECK-GI-NEXT: fmov s19, w1
; CHECK-GI-NEXT: sdiv w3, w3, w4
+; CHECK-GI-NEXT: mov w4, v6.s[3]
; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: sdiv w10, w11, w14
+; CHECK-GI-NEXT: sdiv w10, w4, w5
; CHECK-GI-NEXT: mov v19.s[2], w3
; CHECK-GI-NEXT: mov v19.s[3], w10
; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
@@ -866,13 +866,14 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
;
; CHECK-GI-LABEL: sv32i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 96
+; CHECK-GI-NEXT: sub sp, sp, #112
+; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 112
; CHECK-GI-NEXT: .cfi_offset w19, -8
; CHECK-GI-NEXT: .cfi_offset w20, -16
; CHECK-GI-NEXT: .cfi_offset w21, -24
@@ -901,41 +902,43 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: fmov w9, s7
; CHECK-GI-NEXT: mov w12, v7.s[3]
; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: mov w14, v5.s[1]
; CHECK-GI-NEXT: mov w16, v5.s[3]
; CHECK-GI-NEXT: fmov w6, s19
; CHECK-GI-NEXT: mov w7, v19.s[3]
; CHECK-GI-NEXT: fmov w21, s17
-; CHECK-GI-NEXT: mov w23, v17.s[3]
-; CHECK-GI-NEXT: sdiv w11, w8, w9
+; CHECK-GI-NEXT: sdiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[1]
; CHECK-GI-NEXT: mov w9, v7.s[1]
-; CHECK-GI-NEXT: sdiv w10, w8, w9
+; CHECK-GI-NEXT: mov w22, v17.s[3]
+; CHECK-GI-NEXT: sdiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[2]
; CHECK-GI-NEXT: mov w9, v7.s[2]
-; CHECK-GI-NEXT: mov v20.s[0], w11
+; CHECK-GI-NEXT: fmov s20, w10
; CHECK-GI-NEXT: sdiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[3]
; CHECK-GI-NEXT: sshll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT: mov v20.s[1], w10
+; CHECK-GI-NEXT: mov v20.s[1], w11
; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT: sshll v28.4s, v0.4h, #0
; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
; CHECK-GI-NEXT: sdiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
; CHECK-GI-NEXT: mov v20.s[2], w9
-; CHECK-GI-NEXT: sdiv w15, w12, w13
+; CHECK-GI-NEXT: sdiv w13, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: mov w13, v5.s[1]
-; CHECK-GI-NEXT: mov v20.s[3], w8
-; CHECK-GI-NEXT: sdiv w14, w12, w13
+; CHECK-GI-NEXT: str w8, [sp, #12] // 4-byte Folded Spill
+; CHECK-GI-NEXT: ldr w11, [sp, #12] // 4-byte Folded Reload
+; CHECK-GI-NEXT: mov v20.s[3], w11
+; CHECK-GI-NEXT: sdiv w15, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov w13, v5.s[2]
+; CHECK-GI-NEXT: mov w14, v5.s[2]
; CHECK-GI-NEXT: sshll v5.4s, v6.4h, #0
-; CHECK-GI-NEXT: mov v21.s[0], w15
-; CHECK-GI-NEXT: sdiv w13, w12, w13
+; CHECK-GI-NEXT: fmov s21, w13
+; CHECK-GI-NEXT: sdiv w14, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: sshll2 v4.8h, v2.16b, #0
-; CHECK-GI-NEXT: mov v21.s[1], w14
+; CHECK-GI-NEXT: mov v21.s[1], w15
; CHECK-GI-NEXT: sshll v2.8h, v2.8b, #0
; CHECK-GI-NEXT: sshll v7.4s, v4.4h, #0
; CHECK-GI-NEXT: sshll v30.4s, v2.4h, #0
@@ -944,72 +947,72 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: mls v28.4s, v20.4s, v30.4s
; CHECK-GI-NEXT: sdiv w12, w12, w16
; CHECK-GI-NEXT: fmov w16, s5
-; CHECK-GI-NEXT: mov v21.s[2], w13
-; CHECK-GI-NEXT: sdiv w1, w16, w17
+; CHECK-GI-NEXT: mov v21.s[2], w14
+; CHECK-GI-NEXT: sdiv w18, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[1]
; CHECK-GI-NEXT: mov w17, v7.s[1]
; CHECK-GI-NEXT: mov v21.s[3], w12
; CHECK-GI-NEXT: mls v0.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT: sdiv w0, w16, w17
+; CHECK-GI-NEXT: sdiv w1, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[2]
; CHECK-GI-NEXT: mov w17, v7.s[2]
-; CHECK-GI-NEXT: mov v22.s[0], w1
+; CHECK-GI-NEXT: fmov s22, w18
; CHECK-GI-NEXT: uzp1 v0.8h, v28.8h, v0.8h
-; CHECK-GI-NEXT: sdiv w18, w16, w17
+; CHECK-GI-NEXT: sdiv w0, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[3]
; CHECK-GI-NEXT: mov w17, v7.s[3]
; CHECK-GI-NEXT: sshll2 v5.4s, v6.8h, #0
; CHECK-GI-NEXT: sshll2 v7.4s, v4.8h, #0
-; CHECK-GI-NEXT: mov v22.s[1], w0
+; CHECK-GI-NEXT: mov v22.s[1], w1
; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #0
; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0
; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w4, v7.s[3]
+; CHECK-GI-NEXT: mov w3, v7.s[3]
; CHECK-GI-NEXT: sdiv w16, w16, w17
; CHECK-GI-NEXT: fmov w17, s5
-; CHECK-GI-NEXT: mov v22.s[2], w18
+; CHECK-GI-NEXT: mov v22.s[2], w0
; CHECK-GI-NEXT: sdiv w5, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[1]
; CHECK-GI-NEXT: mov w2, v7.s[1]
; CHECK-GI-NEXT: mov v22.s[3], w16
; CHECK-GI-NEXT: mls v6.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT: sdiv w3, w17, w2
+; CHECK-GI-NEXT: sdiv w4, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[2]
; CHECK-GI-NEXT: mov w2, v7.s[2]
-; CHECK-GI-NEXT: mov v23.s[0], w5
+; CHECK-GI-NEXT: fmov s23, w5
; CHECK-GI-NEXT: sdiv w2, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[3]
-; CHECK-GI-NEXT: mov v23.s[1], w3
-; CHECK-GI-NEXT: sdiv w17, w17, w4
-; CHECK-GI-NEXT: fmov w4, s18
+; CHECK-GI-NEXT: mov v23.s[1], w4
+; CHECK-GI-NEXT: sdiv w17, w17, w3
+; CHECK-GI-NEXT: fmov w3, s18
; CHECK-GI-NEXT: mov v23.s[2], w2
-; CHECK-GI-NEXT: sdiv w20, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[1]
+; CHECK-GI-NEXT: sdiv w20, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[1]
; CHECK-GI-NEXT: mov w6, v19.s[1]
; CHECK-GI-NEXT: mov v23.s[3], w17
; CHECK-GI-NEXT: mls v5.4s, v23.4s, v7.4s
-; CHECK-GI-NEXT: sdiv w19, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[2]
+; CHECK-GI-NEXT: sdiv w19, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[2]
; CHECK-GI-NEXT: mov w6, v19.s[2]
-; CHECK-GI-NEXT: mov v24.s[0], w20
+; CHECK-GI-NEXT: fmov s24, w20
; CHECK-GI-NEXT: uzp1 v2.8h, v6.8h, v5.8h
; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: sdiv w6, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[3]
+; CHECK-GI-NEXT: sdiv w6, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[3]
; CHECK-GI-NEXT: mov v24.s[1], w19
-; CHECK-GI-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: sdiv w4, w4, w7
+; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sdiv w3, w3, w7
; CHECK-GI-NEXT: fmov w7, s16
; CHECK-GI-NEXT: mov v24.s[2], w6
-; CHECK-GI-NEXT: sdiv w24, w7, w21
+; CHECK-GI-NEXT: sdiv w23, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[1]
; CHECK-GI-NEXT: mov w21, v17.s[1]
-; CHECK-GI-NEXT: mov v24.s[3], w4
-; CHECK-GI-NEXT: sdiv w22, w7, w21
+; CHECK-GI-NEXT: mov v24.s[3], w3
+; CHECK-GI-NEXT: sdiv w24, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[2]
; CHECK-GI-NEXT: mov w21, v17.s[2]
; CHECK-GI-NEXT: sshll2 v17.8h, v1.16b, #0
-; CHECK-GI-NEXT: mov v25.s[0], w24
+; CHECK-GI-NEXT: fmov s25, w23
; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: sshll v18.4s, v17.4h, #0
; CHECK-GI-NEXT: sshll v29.4s, v1.4h, #0
@@ -1017,8 +1020,9 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: sdiv w21, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[3]
; CHECK-GI-NEXT: sshll2 v16.8h, v3.16b, #0
-; CHECK-GI-NEXT: mov v25.s[1], w22
+; CHECK-GI-NEXT: mov v25.s[1], w24
; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: sshll v19.4s, v16.4h, #0
; CHECK-GI-NEXT: sshll v31.4s, v3.4h, #0
; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
@@ -1028,51 +1032,51 @@ define <32 x i8> @sv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: mov w28, v19.s[3]
; CHECK-GI-NEXT: sshll2 v19.4s, v16.8h, #0
; CHECK-GI-NEXT: sshll v16.4s, v16.4h, #0
-; CHECK-GI-NEXT: sdiv w7, w7, w23
-; CHECK-GI-NEXT: fmov w23, s18
+; CHECK-GI-NEXT: sdiv w7, w7, w22
+; CHECK-GI-NEXT: fmov w22, s18
; CHECK-GI-NEXT: mov v25.s[2], w21
; CHECK-GI-NEXT: mls v29.4s, v24.4s, v31.4s
-; CHECK-GI-NEXT: ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: fmov w29, s19
; CHECK-GI-NEXT: mov w30, v19.s[1]
-; CHECK-GI-NEXT: mov w15, v19.s[2]
-; CHECK-GI-NEXT: sdiv w25, w23, w25
-; CHECK-GI-NEXT: mov w23, v18.s[1]
+; CHECK-GI-NEXT: mov w8, v19.s[2]
+; CHECK-GI-NEXT: mov w10, v19.s[3]
+; CHECK-GI-NEXT: sdiv w25, w22, w25
+; CHECK-GI-NEXT: mov w22, v18.s[1]
; CHECK-GI-NEXT: mov v25.s[3], w7
; CHECK-GI-NEXT: mls v1.4s, v25.4s, v3.4s
-; CHECK-GI-NEXT: sdiv w26, w23, w26
-; CHECK-GI-NEXT: mov w23, v18.s[2]
-; CHECK-GI-NEXT: mov v26.s[0], w25
+; CHECK-GI-NEXT: sdiv w26, w22, w26
+; CHECK-GI-NEXT: mov w22, v18.s[2]
+; CHECK-GI-NEXT: fmov s26, w25
; CHECK-GI-NEXT: uzp1 v1.8h, v29.8h, v1.8h
-; CHECK-GI-NEXT: sdiv w27, w23, w27
-; CHECK-GI-NEXT: mov w23, v18.s[3]
+; CHECK-GI-NEXT: sdiv w27, w22, w27
+; CHECK-GI-NEXT: mov w22, v18.s[3]
; CHECK-GI-NEXT: sshll2 v18.4s, v17.8h, #0
; CHECK-GI-NEXT: mov v26.s[1], w26
; CHECK-GI-NEXT: sshll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT: ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov w11, v18.s[2]
+; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w9, v18.s[3]
-; CHECK-GI-NEXT: sdiv w23, w23, w28
+; CHECK-GI-NEXT: sdiv w22, w22, w28
; CHECK-GI-NEXT: fmov w28, s18
; CHECK-GI-NEXT: mov v26.s[2], w27
; CHECK-GI-NEXT: sdiv w28, w28, w29
; CHECK-GI-NEXT: mov w29, v18.s[1]
-; CHECK-GI-NEXT: mov v26.s[3], w23
-; CHECK-GI-NEXT: ldp x24, x23, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v26.s[3], w22
+; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: mls v17.4s, v26.4s, v16.4s
; CHECK-GI-NEXT: sdiv w29, w29, w30
-; CHECK-GI-NEXT: mov v27.s[0], w28
-; CHECK-GI-NEXT: ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: sdiv w10, w11, w15
-; CHECK-GI-NEXT: mov w11, v19.s[3]
+; CHECK-GI-NEXT: mov w30, v18.s[2]
+; CHECK-GI-NEXT: fmov s27, w28
+; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sdiv w8, w30, w8
; CHECK-GI-NEXT: mov v27.s[1], w29
-; CHECK-GI-NEXT: sdiv w8, w9, w11
-; CHECK-GI-NEXT: mov v27.s[2], w10
-; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sdiv w9, w9, w10
+; CHECK-GI-NEXT: mov v27.s[2], w8
+; CHECK-GI-NEXT: mov v27.s[3], w9
; CHECK-GI-NEXT: mls v18.4s, v27.4s, v19.4s
; CHECK-GI-NEXT: uzp1 v3.8h, v17.8h, v18.8h
; CHECK-GI-NEXT: uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-GI-NEXT: ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; CHECK-GI-NEXT: add sp, sp, #112
; CHECK-GI-NEXT: ret
entry:
%s = srem <32 x i8> %d, %e
@@ -1109,7 +1113,7 @@ define <2 x i8> @uv2i8(<2 x i8> %d, <2 x i8> %e) {
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -1202,7 +1206,7 @@ define <4 x i8> @uv4i8(<4 x i8> %d, <4 x i8> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -1281,12 +1285,12 @@ define <8 x i8> @uv8i8(<8 x i8> %d, <8 x i8> %e) {
; CHECK-GI-NEXT: fmov w13, s1
; CHECK-GI-NEXT: mov w14, v1.s[1]
; CHECK-GI-NEXT: mov w15, v1.s[2]
+; CHECK-GI-NEXT: mov w16, v1.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v2.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v0.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -1299,11 +1303,11 @@ define <8 x i8> @uv8i8(<8 x i8> %d, <8 x i8> %e) {
; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
; CHECK-GI-NEXT: udiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: udiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v0.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w8, w12
+; CHECK-GI-NEXT: udiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
@@ -1445,20 +1449,20 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: mov w18, v1.s[1]
; CHECK-GI-NEXT: mov w0, v1.s[2]
; CHECK-GI-NEXT: mov w1, v1.s[3]
-; CHECK-GI-NEXT: udiv w11, w8, w9
+; CHECK-GI-NEXT: udiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[1]
; CHECK-GI-NEXT: mov w9, v3.s[1]
; CHECK-GI-NEXT: fmov w2, s7
; CHECK-GI-NEXT: mov w3, v7.s[1]
; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: udiv w10, w8, w9
+; CHECK-GI-NEXT: mov w5, v7.s[3]
+; CHECK-GI-NEXT: udiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[2]
; CHECK-GI-NEXT: mov w9, v3.s[2]
-; CHECK-GI-NEXT: mov v16.s[0], w11
-; CHECK-GI-NEXT: mov w11, v6.s[3]
+; CHECK-GI-NEXT: fmov s16, w10
; CHECK-GI-NEXT: udiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v2.s[3]
-; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: mov v16.s[1], w11
; CHECK-GI-NEXT: udiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
; CHECK-GI-NEXT: mov v16.s[2], w9
@@ -1470,8 +1474,7 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: udiv w15, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[2]
; CHECK-GI-NEXT: mov w13, v5.s[2]
-; CHECK-GI-NEXT: mov v17.s[0], w14
-; CHECK-GI-NEXT: mov w14, v7.s[3]
+; CHECK-GI-NEXT: fmov s17, w14
; CHECK-GI-NEXT: udiv w13, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: mov v17.s[1], w15
@@ -1484,7 +1487,7 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
; CHECK-GI-NEXT: udiv w17, w17, w18
; CHECK-GI-NEXT: mov w18, v0.s[2]
-; CHECK-GI-NEXT: mov v18.s[0], w16
+; CHECK-GI-NEXT: fmov s18, w16
; CHECK-GI-NEXT: udiv w18, w18, w0
; CHECK-GI-NEXT: mov w0, v0.s[3]
; CHECK-GI-NEXT: mov v18.s[1], w17
@@ -1498,10 +1501,11 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
; CHECK-GI-NEXT: udiv w2, w2, w3
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: mov v19.s[0], w1
+; CHECK-GI-NEXT: fmov s19, w1
; CHECK-GI-NEXT: udiv w3, w3, w4
+; CHECK-GI-NEXT: mov w4, v6.s[3]
; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: udiv w10, w11, w14
+; CHECK-GI-NEXT: udiv w10, w4, w5
; CHECK-GI-NEXT: mov v19.s[2], w3
; CHECK-GI-NEXT: mov v19.s[3], w10
; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
@@ -1784,13 +1788,14 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
;
; CHECK-GI-LABEL: uv32i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x28, x27, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x26, x25, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x24, x23, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x22, x21, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 96
+; CHECK-GI-NEXT: sub sp, sp, #112
+; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 112
; CHECK-GI-NEXT: .cfi_offset w19, -8
; CHECK-GI-NEXT: .cfi_offset w20, -16
; CHECK-GI-NEXT: .cfi_offset w21, -24
@@ -1819,41 +1824,43 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: fmov w9, s7
; CHECK-GI-NEXT: mov w12, v7.s[3]
; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: mov w14, v5.s[1]
; CHECK-GI-NEXT: mov w16, v5.s[3]
; CHECK-GI-NEXT: fmov w6, s19
; CHECK-GI-NEXT: mov w7, v19.s[3]
; CHECK-GI-NEXT: fmov w21, s17
-; CHECK-GI-NEXT: mov w23, v17.s[3]
-; CHECK-GI-NEXT: udiv w11, w8, w9
+; CHECK-GI-NEXT: udiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[1]
; CHECK-GI-NEXT: mov w9, v7.s[1]
-; CHECK-GI-NEXT: udiv w10, w8, w9
+; CHECK-GI-NEXT: mov w22, v17.s[3]
+; CHECK-GI-NEXT: udiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[2]
; CHECK-GI-NEXT: mov w9, v7.s[2]
-; CHECK-GI-NEXT: mov v20.s[0], w11
+; CHECK-GI-NEXT: fmov s20, w10
; CHECK-GI-NEXT: udiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[3]
; CHECK-GI-NEXT: ushll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT: mov v20.s[1], w10
+; CHECK-GI-NEXT: mov v20.s[1], w11
; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v28.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
; CHECK-GI-NEXT: udiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
; CHECK-GI-NEXT: mov v20.s[2], w9
-; CHECK-GI-NEXT: udiv w15, w12, w13
+; CHECK-GI-NEXT: udiv w13, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: mov w13, v5.s[1]
-; CHECK-GI-NEXT: mov v20.s[3], w8
-; CHECK-GI-NEXT: udiv w14, w12, w13
+; CHECK-GI-NEXT: str w8, [sp, #12] // 4-byte Folded Spill
+; CHECK-GI-NEXT: ldr w11, [sp, #12] // 4-byte Folded Reload
+; CHECK-GI-NEXT: mov v20.s[3], w11
+; CHECK-GI-NEXT: udiv w15, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov w13, v5.s[2]
+; CHECK-GI-NEXT: mov w14, v5.s[2]
; CHECK-GI-NEXT: ushll v5.4s, v6.4h, #0
-; CHECK-GI-NEXT: mov v21.s[0], w15
-; CHECK-GI-NEXT: udiv w13, w12, w13
+; CHECK-GI-NEXT: fmov s21, w13
+; CHECK-GI-NEXT: udiv w14, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: ushll2 v4.8h, v2.16b, #0
-; CHECK-GI-NEXT: mov v21.s[1], w14
+; CHECK-GI-NEXT: mov v21.s[1], w15
; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-GI-NEXT: ushll v7.4s, v4.4h, #0
; CHECK-GI-NEXT: ushll v30.4s, v2.4h, #0
@@ -1862,72 +1869,72 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: mls v28.4s, v20.4s, v30.4s
; CHECK-GI-NEXT: udiv w12, w12, w16
; CHECK-GI-NEXT: fmov w16, s5
-; CHECK-GI-NEXT: mov v21.s[2], w13
-; CHECK-GI-NEXT: udiv w1, w16, w17
+; CHECK-GI-NEXT: mov v21.s[2], w14
+; CHECK-GI-NEXT: udiv w18, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[1]
; CHECK-GI-NEXT: mov w17, v7.s[1]
; CHECK-GI-NEXT: mov v21.s[3], w12
; CHECK-GI-NEXT: mls v0.4s, v21.4s, v2.4s
-; CHECK-GI-NEXT: udiv w0, w16, w17
+; CHECK-GI-NEXT: udiv w1, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[2]
; CHECK-GI-NEXT: mov w17, v7.s[2]
-; CHECK-GI-NEXT: mov v22.s[0], w1
+; CHECK-GI-NEXT: fmov s22, w18
; CHECK-GI-NEXT: uzp1 v0.8h, v28.8h, v0.8h
-; CHECK-GI-NEXT: udiv w18, w16, w17
+; CHECK-GI-NEXT: udiv w0, w16, w17
; CHECK-GI-NEXT: mov w16, v5.s[3]
; CHECK-GI-NEXT: mov w17, v7.s[3]
; CHECK-GI-NEXT: ushll2 v5.4s, v6.8h, #0
; CHECK-GI-NEXT: ushll2 v7.4s, v4.8h, #0
-; CHECK-GI-NEXT: mov v22.s[1], w0
+; CHECK-GI-NEXT: mov v22.s[1], w1
; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0
; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0
; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w4, v7.s[3]
+; CHECK-GI-NEXT: mov w3, v7.s[3]
; CHECK-GI-NEXT: udiv w16, w16, w17
; CHECK-GI-NEXT: fmov w17, s5
-; CHECK-GI-NEXT: mov v22.s[2], w18
+; CHECK-GI-NEXT: mov v22.s[2], w0
; CHECK-GI-NEXT: udiv w5, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[1]
; CHECK-GI-NEXT: mov w2, v7.s[1]
; CHECK-GI-NEXT: mov v22.s[3], w16
; CHECK-GI-NEXT: mls v6.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT: udiv w3, w17, w2
+; CHECK-GI-NEXT: udiv w4, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[2]
; CHECK-GI-NEXT: mov w2, v7.s[2]
-; CHECK-GI-NEXT: mov v23.s[0], w5
+; CHECK-GI-NEXT: fmov s23, w5
; CHECK-GI-NEXT: udiv w2, w17, w2
; CHECK-GI-NEXT: mov w17, v5.s[3]
-; CHECK-GI-NEXT: mov v23.s[1], w3
-; CHECK-GI-NEXT: udiv w17, w17, w4
-; CHECK-GI-NEXT: fmov w4, s18
+; CHECK-GI-NEXT: mov v23.s[1], w4
+; CHECK-GI-NEXT: udiv w17, w17, w3
+; CHECK-GI-NEXT: fmov w3, s18
; CHECK-GI-NEXT: mov v23.s[2], w2
-; CHECK-GI-NEXT: udiv w20, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[1]
+; CHECK-GI-NEXT: udiv w20, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[1]
; CHECK-GI-NEXT: mov w6, v19.s[1]
; CHECK-GI-NEXT: mov v23.s[3], w17
; CHECK-GI-NEXT: mls v5.4s, v23.4s, v7.4s
-; CHECK-GI-NEXT: udiv w19, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[2]
+; CHECK-GI-NEXT: udiv w19, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[2]
; CHECK-GI-NEXT: mov w6, v19.s[2]
-; CHECK-GI-NEXT: mov v24.s[0], w20
+; CHECK-GI-NEXT: fmov s24, w20
; CHECK-GI-NEXT: uzp1 v2.8h, v6.8h, v5.8h
; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: udiv w6, w4, w6
-; CHECK-GI-NEXT: mov w4, v18.s[3]
+; CHECK-GI-NEXT: udiv w6, w3, w6
+; CHECK-GI-NEXT: mov w3, v18.s[3]
; CHECK-GI-NEXT: mov v24.s[1], w19
-; CHECK-GI-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: udiv w4, w4, w7
+; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-GI-NEXT: udiv w3, w3, w7
; CHECK-GI-NEXT: fmov w7, s16
; CHECK-GI-NEXT: mov v24.s[2], w6
-; CHECK-GI-NEXT: udiv w24, w7, w21
+; CHECK-GI-NEXT: udiv w23, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[1]
; CHECK-GI-NEXT: mov w21, v17.s[1]
-; CHECK-GI-NEXT: mov v24.s[3], w4
-; CHECK-GI-NEXT: udiv w22, w7, w21
+; CHECK-GI-NEXT: mov v24.s[3], w3
+; CHECK-GI-NEXT: udiv w24, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[2]
; CHECK-GI-NEXT: mov w21, v17.s[2]
; CHECK-GI-NEXT: ushll2 v17.8h, v1.16b, #0
-; CHECK-GI-NEXT: mov v25.s[0], w24
+; CHECK-GI-NEXT: fmov s25, w23
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: ushll v18.4s, v17.4h, #0
; CHECK-GI-NEXT: ushll v29.4s, v1.4h, #0
@@ -1935,8 +1942,9 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: udiv w21, w7, w21
; CHECK-GI-NEXT: mov w7, v16.s[3]
; CHECK-GI-NEXT: ushll2 v16.8h, v3.16b, #0
-; CHECK-GI-NEXT: mov v25.s[1], w22
+; CHECK-GI-NEXT: mov v25.s[1], w24
; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: ushll v19.4s, v16.4h, #0
; CHECK-GI-NEXT: ushll v31.4s, v3.4h, #0
; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0
@@ -1946,51 +1954,51 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: mov w28, v19.s[3]
; CHECK-GI-NEXT: ushll2 v19.4s, v16.8h, #0
; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0
-; CHECK-GI-NEXT: udiv w7, w7, w23
-; CHECK-GI-NEXT: fmov w23, s18
+; CHECK-GI-NEXT: udiv w7, w7, w22
+; CHECK-GI-NEXT: fmov w22, s18
; CHECK-GI-NEXT: mov v25.s[2], w21
; CHECK-GI-NEXT: mls v29.4s, v24.4s, v31.4s
-; CHECK-GI-NEXT: ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT: fmov w29, s19
; CHECK-GI-NEXT: mov w30, v19.s[1]
-; CHECK-GI-NEXT: mov w15, v19.s[2]
-; CHECK-GI-NEXT: udiv w25, w23, w25
-; CHECK-GI-NEXT: mov w23, v18.s[1]
+; CHECK-GI-NEXT: mov w8, v19.s[2]
+; CHECK-GI-NEXT: mov w10, v19.s[3]
+; CHECK-GI-NEXT: udiv w25, w22, w25
+; CHECK-GI-NEXT: mov w22, v18.s[1]
; CHECK-GI-NEXT: mov v25.s[3], w7
; CHECK-GI-NEXT: mls v1.4s, v25.4s, v3.4s
-; CHECK-GI-NEXT: udiv w26, w23, w26
-; CHECK-GI-NEXT: mov w23, v18.s[2]
-; CHECK-GI-NEXT: mov v26.s[0], w25
+; CHECK-GI-NEXT: udiv w26, w22, w26
+; CHECK-GI-NEXT: mov w22, v18.s[2]
+; CHECK-GI-NEXT: fmov s26, w25
; CHECK-GI-NEXT: uzp1 v1.8h, v29.8h, v1.8h
-; CHECK-GI-NEXT: udiv w27, w23, w27
-; CHECK-GI-NEXT: mov w23, v18.s[3]
+; CHECK-GI-NEXT: udiv w27, w22, w27
+; CHECK-GI-NEXT: mov w22, v18.s[3]
; CHECK-GI-NEXT: ushll2 v18.4s, v17.8h, #0
; CHECK-GI-NEXT: mov v26.s[1], w26
; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT: ldp x26, x25, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov w11, v18.s[2]
+; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT: mov w9, v18.s[3]
-; CHECK-GI-NEXT: udiv w23, w23, w28
+; CHECK-GI-NEXT: udiv w22, w22, w28
; CHECK-GI-NEXT: fmov w28, s18
; CHECK-GI-NEXT: mov v26.s[2], w27
; CHECK-GI-NEXT: udiv w28, w28, w29
; CHECK-GI-NEXT: mov w29, v18.s[1]
-; CHECK-GI-NEXT: mov v26.s[3], w23
-; CHECK-GI-NEXT: ldp x24, x23, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v26.s[3], w22
+; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT: mls v17.4s, v26.4s, v16.4s
; CHECK-GI-NEXT: udiv w29, w29, w30
-; CHECK-GI-NEXT: mov v27.s[0], w28
-; CHECK-GI-NEXT: ldp x28, x27, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: udiv w10, w11, w15
-; CHECK-GI-NEXT: mov w11, v19.s[3]
+; CHECK-GI-NEXT: mov w30, v18.s[2]
+; CHECK-GI-NEXT: fmov s27, w28
+; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: udiv w8, w30, w8
; CHECK-GI-NEXT: mov v27.s[1], w29
-; CHECK-GI-NEXT: udiv w8, w9, w11
-; CHECK-GI-NEXT: mov v27.s[2], w10
-; CHECK-GI-NEXT: mov v27.s[3], w8
+; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: udiv w9, w9, w10
+; CHECK-GI-NEXT: mov v27.s[2], w8
+; CHECK-GI-NEXT: mov v27.s[3], w9
; CHECK-GI-NEXT: mls v18.4s, v27.4s, v19.4s
; CHECK-GI-NEXT: uzp1 v3.8h, v17.8h, v18.8h
; CHECK-GI-NEXT: uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-GI-NEXT: ldp x29, x30, [sp], #96 // 16-byte Folded Reload
+; CHECK-GI-NEXT: add sp, sp, #112
; CHECK-GI-NEXT: ret
entry:
%s = urem <32 x i8> %d, %e
@@ -2029,7 +2037,7 @@ define <2 x i16> @sv2i16(<2 x i16> %d, <2 x i16> %e) {
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -2129,7 +2137,7 @@ define <4 x i16> @sv4i16(<4 x i16> %d, <4 x i16> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -2204,12 +2212,12 @@ define <8 x i16> @sv8i16(<8 x i16> %d, <8 x i16> %e) {
; CHECK-GI-NEXT: fmov w13, s1
; CHECK-GI-NEXT: mov w14, v1.s[1]
; CHECK-GI-NEXT: mov w15, v1.s[2]
+; CHECK-GI-NEXT: mov w16, v1.s[3]
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v2.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v0.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -2222,11 +2230,11 @@ define <8 x i16> @sv8i16(<8 x i16> %d, <8 x i16> %e) {
; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
; CHECK-GI-NEXT: sdiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: sdiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v0.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: sdiv w8, w8, w12
+; CHECK-GI-NEXT: sdiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
@@ -2387,17 +2395,18 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mov w1, v7.s[3]
; CHECK-GI-NEXT: sshll2 v7.4s, v3.8h, #0
; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
-; CHECK-GI-NEXT: sdiv w11, w8, w9
+; CHECK-GI-NEXT: sdiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[1]
; CHECK-GI-NEXT: mov w9, v5.s[1]
; CHECK-GI-NEXT: fmov w2, s7
; CHECK-GI-NEXT: mov w3, v7.s[1]
; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: sdiv w10, w8, w9
+; CHECK-GI-NEXT: mov w5, v7.s[3]
+; CHECK-GI-NEXT: sdiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[2]
; CHECK-GI-NEXT: mov w9, v5.s[2]
; CHECK-GI-NEXT: sshll2 v5.4s, v2.8h, #0
-; CHECK-GI-NEXT: mov v16.s[0], w11
+; CHECK-GI-NEXT: fmov s16, w10
; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
; CHECK-GI-NEXT: fmov w13, s5
; CHECK-GI-NEXT: mov w14, v5.s[1]
@@ -2406,7 +2415,7 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: sdiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[3]
; CHECK-GI-NEXT: sshll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: mov v16.s[1], w11
; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
; CHECK-GI-NEXT: sdiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
@@ -2417,8 +2426,7 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v0.4s, v16.4s, v2.4s
; CHECK-GI-NEXT: sdiv w14, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov v17.s[0], w13
-; CHECK-GI-NEXT: mov w13, v7.s[3]
+; CHECK-GI-NEXT: fmov s17, w13
; CHECK-GI-NEXT: sdiv w15, w12, w15
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: mov v17.s[1], w14
@@ -2431,14 +2439,13 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
; CHECK-GI-NEXT: sdiv w17, w17, w18
; CHECK-GI-NEXT: mov w18, v6.s[2]
-; CHECK-GI-NEXT: mov v18.s[0], w16
+; CHECK-GI-NEXT: fmov s18, w16
; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v4.8h
; CHECK-GI-NEXT: sdiv w18, w18, w0
; CHECK-GI-NEXT: mov w0, v6.s[3]
; CHECK-GI-NEXT: sshll2 v6.4s, v1.8h, #0
; CHECK-GI-NEXT: mov v18.s[1], w17
; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT: mov w11, v6.s[3]
; CHECK-GI-NEXT: sdiv w0, w0, w1
; CHECK-GI-NEXT: fmov w1, s6
; CHECK-GI-NEXT: mov v18.s[2], w18
@@ -2448,10 +2455,11 @@ define <16 x i16> @sv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v1.4s, v18.4s, v3.4s
; CHECK-GI-NEXT: sdiv w2, w2, w3
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: mov v19.s[0], w1
+; CHECK-GI-NEXT: fmov s19, w1
; CHECK-GI-NEXT: sdiv w3, w3, w4
+; CHECK-GI-NEXT: mov w4, v6.s[3]
; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: sdiv w10, w11, w13
+; CHECK-GI-NEXT: sdiv w10, w4, w5
; CHECK-GI-NEXT: mov v19.s[2], w3
; CHECK-GI-NEXT: mov v19.s[3], w10
; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
@@ -2492,7 +2500,7 @@ define <2 x i16> @uv2i16(<2 x i16> %d, <2 x i16> %e) {
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -2597,7 +2605,7 @@ define <4 x i16> @uv4i16(<4 x i16> %d, <4 x i16> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -2672,12 +2680,12 @@ define <8 x i16> @uv8i16(<8 x i16> %d, <8 x i16> %e) {
; CHECK-GI-NEXT: fmov w13, s1
; CHECK-GI-NEXT: mov w14, v1.s[1]
; CHECK-GI-NEXT: mov w15, v1.s[2]
+; CHECK-GI-NEXT: mov w16, v1.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v2.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v0.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -2690,11 +2698,11 @@ define <8 x i16> @uv8i16(<8 x i16> %d, <8 x i16> %e) {
; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
; CHECK-GI-NEXT: udiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: udiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v0.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w8, w12
+; CHECK-GI-NEXT: udiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
@@ -2855,17 +2863,18 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mov w1, v7.s[3]
; CHECK-GI-NEXT: ushll2 v7.4s, v3.8h, #0
; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-GI-NEXT: udiv w11, w8, w9
+; CHECK-GI-NEXT: udiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[1]
; CHECK-GI-NEXT: mov w9, v5.s[1]
; CHECK-GI-NEXT: fmov w2, s7
; CHECK-GI-NEXT: mov w3, v7.s[1]
; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: udiv w10, w8, w9
+; CHECK-GI-NEXT: mov w5, v7.s[3]
+; CHECK-GI-NEXT: udiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[2]
; CHECK-GI-NEXT: mov w9, v5.s[2]
; CHECK-GI-NEXT: ushll2 v5.4s, v2.8h, #0
-; CHECK-GI-NEXT: mov v16.s[0], w11
+; CHECK-GI-NEXT: fmov s16, w10
; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-GI-NEXT: fmov w13, s5
; CHECK-GI-NEXT: mov w14, v5.s[1]
@@ -2874,7 +2883,7 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: udiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[3]
; CHECK-GI-NEXT: ushll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT: mov v16.s[1], w10
+; CHECK-GI-NEXT: mov v16.s[1], w11
; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-GI-NEXT: udiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
@@ -2885,8 +2894,7 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v0.4s, v16.4s, v2.4s
; CHECK-GI-NEXT: udiv w14, w12, w14
; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov v17.s[0], w13
-; CHECK-GI-NEXT: mov w13, v7.s[3]
+; CHECK-GI-NEXT: fmov s17, w13
; CHECK-GI-NEXT: udiv w15, w12, w15
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: mov v17.s[1], w14
@@ -2899,14 +2907,13 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
; CHECK-GI-NEXT: udiv w17, w17, w18
; CHECK-GI-NEXT: mov w18, v6.s[2]
-; CHECK-GI-NEXT: mov v18.s[0], w16
+; CHECK-GI-NEXT: fmov s18, w16
; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v4.8h
; CHECK-GI-NEXT: udiv w18, w18, w0
; CHECK-GI-NEXT: mov w0, v6.s[3]
; CHECK-GI-NEXT: ushll2 v6.4s, v1.8h, #0
; CHECK-GI-NEXT: mov v18.s[1], w17
; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT: mov w11, v6.s[3]
; CHECK-GI-NEXT: udiv w0, w0, w1
; CHECK-GI-NEXT: fmov w1, s6
; CHECK-GI-NEXT: mov v18.s[2], w18
@@ -2916,10 +2923,11 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI-NEXT: mls v1.4s, v18.4s, v3.4s
; CHECK-GI-NEXT: udiv w2, w2, w3
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: mov v19.s[0], w1
+; CHECK-GI-NEXT: fmov s19, w1
; CHECK-GI-NEXT: udiv w3, w3, w4
+; CHECK-GI-NEXT: mov w4, v6.s[3]
; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: udiv w10, w11, w13
+; CHECK-GI-NEXT: udiv w10, w4, w5
; CHECK-GI-NEXT: mov v19.s[2], w3
; CHECK-GI-NEXT: mov v19.s[3], w10
; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
@@ -2958,7 +2966,7 @@ define <2 x i32> @sv2i32(<2 x i32> %d, <2 x i32> %e) {
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -2990,10 +2998,10 @@ define <3 x i32> @sv3i32(<3 x i32> %d, <3 x i32> %e) {
;
; CHECK-GI-LABEL: sv3i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v1.s[1]
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: mov s0, v0.s[2]
; CHECK-GI-NEXT: mov s1, v1.s[2]
; CHECK-GI-NEXT: sdiv w10, w8, w9
@@ -3003,11 +3011,11 @@ define <3 x i32> @sv3i32(<3 x i32> %d, <3 x i32> %e) {
; CHECK-GI-NEXT: fmov w15, s1
; CHECK-GI-NEXT: sdiv w13, w11, w12
; CHECK-GI-NEXT: msub w8, w10, w9, w8
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: sdiv w9, w14, w15
-; CHECK-GI-NEXT: msub w8, w13, w12, w11
-; CHECK-GI-NEXT: mov v0.s[1], w8
-; CHECK-GI-NEXT: msub w8, w9, w15, w14
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: sdiv w16, w14, w15
+; CHECK-GI-NEXT: msub w9, w13, w12, w11
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: msub w8, w16, w15, w14
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -3051,7 +3059,7 @@ define <4 x i32> @sv4i32(<4 x i32> %d, <4 x i32> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -3129,12 +3137,12 @@ define <8 x i32> @sv8i32(<8 x i32> %d, <8 x i32> %e) {
; CHECK-GI-NEXT: fmov w13, s3
; CHECK-GI-NEXT: mov w14, v3.s[1]
; CHECK-GI-NEXT: mov w15, v3.s[2]
+; CHECK-GI-NEXT: mov w16, v3.s[3]
; CHECK-GI-NEXT: sdiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: sdiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v1.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: sdiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -3147,11 +3155,11 @@ define <8 x i32> @sv8i32(<8 x i32> %d, <8 x i32> %e) {
; CHECK-GI-NEXT: mls v0.4s, v4.4s, v2.4s
; CHECK-GI-NEXT: sdiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v1.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v3.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: sdiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v1.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: sdiv w8, w8, w12
+; CHECK-GI-NEXT: sdiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v1.4s, v5.4s, v3.4s
@@ -3189,7 +3197,7 @@ define <2 x i32> @uv2i32(<2 x i32> %d, <2 x i32> %e) {
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: mls v0.2s, v2.2s, v1.2s
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -3221,10 +3229,10 @@ define <3 x i32> @uv3i32(<3 x i32> %d, <3 x i32> %e) {
;
; CHECK-GI-LABEL: uv3i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: mov s2, v0.s[1]
; CHECK-GI-NEXT: mov s3, v1.s[1]
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: mov s0, v0.s[2]
; CHECK-GI-NEXT: mov s1, v1.s[2]
; CHECK-GI-NEXT: udiv w10, w8, w9
@@ -3234,11 +3242,11 @@ define <3 x i32> @uv3i32(<3 x i32> %d, <3 x i32> %e) {
; CHECK-GI-NEXT: fmov w15, s1
; CHECK-GI-NEXT: udiv w13, w11, w12
; CHECK-GI-NEXT: msub w8, w10, w9, w8
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: udiv w9, w14, w15
-; CHECK-GI-NEXT: msub w8, w13, w12, w11
-; CHECK-GI-NEXT: mov v0.s[1], w8
-; CHECK-GI-NEXT: msub w8, w9, w15, w14
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: udiv w16, w14, w15
+; CHECK-GI-NEXT: msub w9, w13, w12, w11
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: msub w8, w16, w15, w14
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -3282,7 +3290,7 @@ define <4 x i32> @uv4i32(<4 x i32> %d, <4 x i32> %e) {
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v2.s[1], w9
@@ -3360,12 +3368,12 @@ define <8 x i32> @uv8i32(<8 x i32> %d, <8 x i32> %e) {
; CHECK-GI-NEXT: fmov w13, s3
; CHECK-GI-NEXT: mov w14, v3.s[1]
; CHECK-GI-NEXT: mov w15, v3.s[2]
+; CHECK-GI-NEXT: mov w16, v3.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
; CHECK-GI-NEXT: mov w9, v0.s[1]
; CHECK-GI-NEXT: udiv w9, w9, w10
; CHECK-GI-NEXT: mov w10, v0.s[2]
-; CHECK-GI-NEXT: mov v4.s[0], w8
-; CHECK-GI-NEXT: mov w8, v1.s[3]
+; CHECK-GI-NEXT: fmov s4, w8
; CHECK-GI-NEXT: udiv w10, w10, w11
; CHECK-GI-NEXT: mov w11, v0.s[3]
; CHECK-GI-NEXT: mov v4.s[1], w9
@@ -3378,11 +3386,11 @@ define <8 x i32> @uv8i32(<8 x i32> %d, <8 x i32> %e) {
; CHECK-GI-NEXT: mls v0.4s, v4.4s, v2.4s
; CHECK-GI-NEXT: udiv w13, w13, w14
; CHECK-GI-NEXT: mov w14, v1.s[2]
-; CHECK-GI-NEXT: mov v5.s[0], w12
-; CHECK-GI-NEXT: mov w12, v3.s[3]
+; CHECK-GI-NEXT: fmov s5, w12
; CHECK-GI-NEXT: udiv w14, w14, w15
+; CHECK-GI-NEXT: mov w15, v1.s[3]
; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w8, w12
+; CHECK-GI-NEXT: udiv w8, w15, w16
; CHECK-GI-NEXT: mov v5.s[2], w14
; CHECK-GI-NEXT: mov v5.s[3], w8
; CHECK-GI-NEXT: mls v1.4s, v5.4s, v3.4s
@@ -3415,14 +3423,14 @@ define <2 x i64> @sv2i64(<2 x i64> %d, <2 x i64> %e) {
; CHECK-GI-NEXT: mov x11, v0.d[1]
; CHECK-GI-NEXT: sdiv x8, x8, x9
; CHECK-GI-NEXT: sdiv x11, x11, x10
-; CHECK-GI-NEXT: mov v1.d[0], x8
+; CHECK-GI-NEXT: fmov d1, x8
; CHECK-GI-NEXT: mov v1.d[1], x11
-; CHECK-GI-NEXT: fmov x8, d1
-; CHECK-GI-NEXT: mov x11, v1.d[1]
-; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x11, x10
-; CHECK-GI-NEXT: mov v1.d[0], x8
-; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v1.d[1]
+; CHECK-GI-NEXT: mul x9, x11, x9
+; CHECK-GI-NEXT: mul x8, x8, x10
+; CHECK-GI-NEXT: fmov d1, x9
+; CHECK-GI-NEXT: mov v1.d[1], x8
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: ret
entry:
@@ -3469,21 +3477,21 @@ define <3 x i64> @sv3i64(<3 x i64> %d, <3 x i64> %e) {
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT: sdiv x8, x8, x9
; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: fmov x11, d3
-; CHECK-GI-NEXT: mov x14, v3.d[1]
+; CHECK-GI-NEXT: fmov x14, d3
+; CHECK-GI-NEXT: mov x12, v3.d[1]
; CHECK-GI-NEXT: sdiv x9, x9, x10
-; CHECK-GI-NEXT: mov v6.d[0], x8
+; CHECK-GI-NEXT: fmov d6, x8
; CHECK-GI-NEXT: fmov x8, d2
; CHECK-GI-NEXT: mov v6.d[1], x9
; CHECK-GI-NEXT: fmov x9, d5
-; CHECK-GI-NEXT: sdiv x12, x8, x9
-; CHECK-GI-NEXT: fmov x10, d6
-; CHECK-GI-NEXT: mov x13, v6.d[1]
-; CHECK-GI-NEXT: mul x10, x10, x11
-; CHECK-GI-NEXT: mul x11, x13, x14
-; CHECK-GI-NEXT: mov v2.d[0], x10
+; CHECK-GI-NEXT: sdiv x10, x8, x9
+; CHECK-GI-NEXT: fmov x13, d6
+; CHECK-GI-NEXT: mov x11, v6.d[1]
+; CHECK-GI-NEXT: mul x13, x13, x14
+; CHECK-GI-NEXT: mul x11, x11, x12
+; CHECK-GI-NEXT: fmov d2, x13
; CHECK-GI-NEXT: mov v2.d[1], x11
-; CHECK-GI-NEXT: msub x8, x12, x9, x8
+; CHECK-GI-NEXT: msub x8, x10, x9, x8
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -3530,26 +3538,26 @@ define <4 x i64> @sv4i64(<4 x i64> %d, <4 x i64> %e) {
; CHECK-GI-NEXT: mov x14, v3.d[1]
; CHECK-GI-NEXT: mov x15, v1.d[1]
; CHECK-GI-NEXT: sdiv x8, x8, x9
-; CHECK-GI-NEXT: sdiv x12, x12, x13
-; CHECK-GI-NEXT: mov v2.d[0], x8
; CHECK-GI-NEXT: sdiv x11, x11, x10
-; CHECK-GI-NEXT: mov v3.d[0], x12
-; CHECK-GI-NEXT: sdiv x15, x15, x14
+; CHECK-GI-NEXT: fmov d2, x8
+; CHECK-GI-NEXT: sdiv x12, x12, x13
; CHECK-GI-NEXT: mov v2.d[1], x11
-; CHECK-GI-NEXT: fmov x8, d2
-; CHECK-GI-NEXT: mov x11, v2.d[1]
-; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x10, x11, x10
-; CHECK-GI-NEXT: mov v2.d[0], x8
-; CHECK-GI-NEXT: mov v3.d[1], x15
-; CHECK-GI-NEXT: mov v2.d[1], x10
-; CHECK-GI-NEXT: fmov x9, d3
-; CHECK-GI-NEXT: mov x12, v3.d[1]
+; CHECK-GI-NEXT: fmov x11, d2
+; CHECK-GI-NEXT: mov x8, v2.d[1]
+; CHECK-GI-NEXT: mul x9, x11, x9
+; CHECK-GI-NEXT: mul x8, x8, x10
+; CHECK-GI-NEXT: fmov d2, x9
+; CHECK-GI-NEXT: mov v2.d[1], x8
+; CHECK-GI-NEXT: sdiv x15, x15, x14
+; CHECK-GI-NEXT: fmov d3, x12
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v2.2d
-; CHECK-GI-NEXT: mul x9, x9, x13
-; CHECK-GI-NEXT: mul x11, x12, x14
-; CHECK-GI-NEXT: mov v3.d[0], x9
-; CHECK-GI-NEXT: mov v3.d[1], x11
+; CHECK-GI-NEXT: mov v3.d[1], x15
+; CHECK-GI-NEXT: fmov x11, d3
+; CHECK-GI-NEXT: mov x10, v3.d[1]
+; CHECK-GI-NEXT: mul x11, x11, x13
+; CHECK-GI-NEXT: mul x10, x10, x14
+; CHECK-GI-NEXT: fmov d3, x11
+; CHECK-GI-NEXT: mov v3.d[1], x10
; CHECK-GI-NEXT: sub v1.2d, v1.2d, v3.2d
; CHECK-GI-NEXT: ret
entry:
@@ -3580,14 +3588,14 @@ define <2 x i64> @uv2i64(<2 x i64> %d, <2 x i64> %e) {
; CHECK-GI-NEXT: mov x11, v0.d[1]
; CHECK-GI-NEXT: udiv x8, x8, x9
; CHECK-GI-NEXT: udiv x11, x11, x10
-; CHECK-GI-NEXT: mov v1.d[0], x8
+; CHECK-GI-NEXT: fmov d1, x8
; CHECK-GI-NEXT: mov v1.d[1], x11
-; CHECK-GI-NEXT: fmov x8, d1
-; CHECK-GI-NEXT: mov x11, v1.d[1]
-; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x9, x11, x10
-; CHECK-GI-NEXT: mov v1.d[0], x8
-; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: mov x8, v1.d[1]
+; CHECK-GI-NEXT: mul x9, x11, x9
+; CHECK-GI-NEXT: mul x8, x8, x10
+; CHECK-GI-NEXT: fmov d1, x9
+; CHECK-GI-NEXT: mov v1.d[1], x8
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: ret
entry:
@@ -3634,21 +3642,21 @@ define <3 x i64> @uv3i64(<3 x i64> %d, <3 x i64> %e) {
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT: udiv x8, x8, x9
; CHECK-GI-NEXT: fmov x9, d1
-; CHECK-GI-NEXT: fmov x11, d3
-; CHECK-GI-NEXT: mov x14, v3.d[1]
+; CHECK-GI-NEXT: fmov x14, d3
+; CHECK-GI-NEXT: mov x12, v3.d[1]
; CHECK-GI-NEXT: udiv x9, x9, x10
-; CHECK-GI-NEXT: mov v6.d[0], x8
+; CHECK-GI-NEXT: fmov d6, x8
; CHECK-GI-NEXT: fmov x8, d2
; CHECK-GI-NEXT: mov v6.d[1], x9
; CHECK-GI-NEXT: fmov x9, d5
-; CHECK-GI-NEXT: udiv x12, x8, x9
-; CHECK-GI-NEXT: fmov x10, d6
-; CHECK-GI-NEXT: mov x13, v6.d[1]
-; CHECK-GI-NEXT: mul x10, x10, x11
-; CHECK-GI-NEXT: mul x11, x13, x14
-; CHECK-GI-NEXT: mov v2.d[0], x10
+; CHECK-GI-NEXT: udiv x10, x8, x9
+; CHECK-GI-NEXT: fmov x13, d6
+; CHECK-GI-NEXT: mov x11, v6.d[1]
+; CHECK-GI-NEXT: mul x13, x13, x14
+; CHECK-GI-NEXT: mul x11, x11, x12
+; CHECK-GI-NEXT: fmov d2, x13
; CHECK-GI-NEXT: mov v2.d[1], x11
-; CHECK-GI-NEXT: msub x8, x12, x9, x8
+; CHECK-GI-NEXT: msub x8, x10, x9, x8
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
@@ -3695,26 +3703,26 @@ define <4 x i64> @uv4i64(<4 x i64> %d, <4 x i64> %e) {
; CHECK-GI-NEXT: mov x14, v3.d[1]
; CHECK-GI-NEXT: mov x15, v1.d[1]
; CHECK-GI-NEXT: udiv x8, x8, x9
-; CHECK-GI-NEXT: udiv x12, x12, x13
-; CHECK-GI-NEXT: mov v2.d[0], x8
; CHECK-GI-NEXT: udiv x11, x11, x10
-; CHECK-GI-NEXT: mov v3.d[0], x12
-; CHECK-GI-NEXT: udiv x15, x15, x14
+; CHECK-GI-NEXT: fmov d2, x8
+; CHECK-GI-NEXT: udiv x12, x12, x13
; CHECK-GI-NEXT: mov v2.d[1], x11
-; CHECK-GI-NEXT: fmov x8, d2
-; CHECK-GI-NEXT: mov x11, v2.d[1]
-; CHECK-GI-NEXT: mul x8, x8, x9
-; CHECK-GI-NEXT: mul x10, x11, x10
-; CHECK-GI-NEXT: mov v2.d[0], x8
-; CHECK-GI-NEXT: mov v3.d[1], x15
-; CHECK-GI-NEXT: mov v2.d[1], x10
-; CHECK-GI-NEXT: fmov x9, d3
-; CHECK-GI-NEXT: mov x12, v3.d[1]
+; CHECK-GI-NEXT: fmov x11, d2
+; CHECK-GI-NEXT: mov x8, v2.d[1]
+; CHECK-GI-NEXT: mul x9, x11, x9
+; CHECK-GI-NEXT: mul x8, x8, x10
+; CHECK-GI-NEXT: fmov d2, x9
+; CHECK-GI-NEXT: mov v2.d[1], x8
+; CHECK-GI-NEXT: udiv x15, x15, x14
+; CHECK-GI-NEXT: fmov d3, x12
; CHECK-GI-NEXT: sub v0.2d, v0.2d, v2.2d
-; CHECK-GI-NEXT: mul x9, x9, x13
-; CHECK-GI-NEXT: mul x11, x12, x14
-; CHECK-GI-NEXT: mov v3.d[0], x9
-; CHECK-GI-NEXT: mov v3.d[1], x11
+; CHECK-GI-NEXT: mov v3.d[1], x15
+; CHECK-GI-NEXT: fmov x11, d3
+; CHECK-GI-NEXT: mov x10, v3.d[1]
+; CHECK-GI-NEXT: mul x11, x11, x13
+; CHECK-GI-NEXT: mul x10, x10, x14
+; CHECK-GI-NEXT: fmov d3, x11
+; CHECK-GI-NEXT: mov v3.d[1], x10
; CHECK-GI-NEXT: sub v1.2d, v1.2d, v3.2d
; CHECK-GI-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/select_cc.ll b/llvm/test/CodeGen/AArch64/select_cc.ll
index 66dbd4ed5a4dc..73e4d4c7f0aeb 100644
--- a/llvm/test/CodeGen/AArch64/select_cc.ll
+++ b/llvm/test/CodeGen/AArch64/select_cc.ll
@@ -129,7 +129,7 @@ define <4 x i32> @select_icmp_sgt(<4 x i32> %a, <4 x i8> %b) {
; CHECK-GI-NEXT: cmgt v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT: umov w8, v1.b[0]
; CHECK-GI-NEXT: umov w9, v1.b[1]
-; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: fmov s2, w8
; CHECK-GI-NEXT: umov w8, v1.b[2]
; CHECK-GI-NEXT: mov v2.s[1], w9
; CHECK-GI-NEXT: umov w9, v1.b[3]
diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll
index 2f193cefd7c0c..ef6b65cd50a1e 100644
--- a/llvm/test/CodeGen/AArch64/sext.ll
+++ b/llvm/test/CodeGen/AArch64/sext.ll
@@ -249,10 +249,10 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-GI-LABEL: sext_v3i8_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sxtb w8, w0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: sxtb w8, w1
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: sxtb w9, w1
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: sxtb w8, w2
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -306,7 +306,7 @@ define <3 x i32> @sext_v3i16_v3i32(<3 x i16> %a) {
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT: smov w8, v0.h[0]
; CHECK-GI-NEXT: smov w9, v0.h[1]
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: smov w8, v0.h[2]
; CHECK-GI-NEXT: mov v1.s[1], w9
; CHECK-GI-NEXT: mov v1.s[2], w8
@@ -411,10 +411,10 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-GI-LABEL: sext_v3i10_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sbfx w8, w0, #0, #10
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: sbfx w8, w1, #0, #10
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: sbfx w9, w1, #0, #10
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: sbfx w8, w2, #0, #10
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -1192,50 +1192,50 @@ define <16 x i64> @sext_v16i10_v16i64(<16 x i10> %a) {
;
; CHECK-GI-LABEL: sext_v16i10_v16i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v1.s[0], w0
-; CHECK-GI-NEXT: mov v2.s[0], w2
+; CHECK-GI-NEXT: fmov s7, w0
+; CHECK-GI-NEXT: fmov s17, w2
; CHECK-GI-NEXT: ldr s0, [sp]
-; CHECK-GI-NEXT: mov v3.s[0], w4
-; CHECK-GI-NEXT: mov v4.s[0], w6
-; CHECK-GI-NEXT: ldr s5, [sp, #8]
-; CHECK-GI-NEXT: ldr s6, [sp, #16]
-; CHECK-GI-NEXT: ldr s7, [sp, #24]
-; CHECK-GI-NEXT: ldr s16, [sp, #32]
-; CHECK-GI-NEXT: ldr s17, [sp, #40]
-; CHECK-GI-NEXT: ldr s18, [sp, #48]
-; CHECK-GI-NEXT: ldr s19, [sp, #56]
-; CHECK-GI-NEXT: mov v1.s[1], w1
-; CHECK-GI-NEXT: mov v0.s[1], v5.s[0]
-; CHECK-GI-NEXT: mov v2.s[1], w3
-; CHECK-GI-NEXT: mov v3.s[1], w5
-; CHECK-GI-NEXT: mov v4.s[1], w7
-; CHECK-GI-NEXT: mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT: mov v16.s[1], v17.s[0]
-; CHECK-GI-NEXT: mov v18.s[1], v19.s[0]
+; CHECK-GI-NEXT: fmov s18, w4
+; CHECK-GI-NEXT: fmov s19, w6
+; CHECK-GI-NEXT: ldr s1, [sp, #8]
+; CHECK-GI-NEXT: ldr s2, [sp, #16]
+; CHECK-GI-NEXT: ldr s3, [sp, #24]
+; CHECK-GI-NEXT: ldr s4, [sp, #32]
+; CHECK-GI-NEXT: ldr s5, [sp, #40]
+; CHECK-GI-NEXT: ldr s6, [sp, #48]
+; CHECK-GI-NEXT: ldr s16, [sp, #56]
+; CHECK-GI-NEXT: mov v7.s[1], w1
+; CHECK-GI-NEXT: mov v17.s[1], w3
+; CHECK-GI-NEXT: mov v18.s[1], w5
+; CHECK-GI-NEXT: mov v19.s[1], w7
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT: mov v2.s[1], v3.s[0]
+; CHECK-GI-NEXT: mov v4.s[1], v5.s[0]
+; CHECK-GI-NEXT: mov v6.s[1], v16.s[0]
+; CHECK-GI-NEXT: ushll v1.2d, v7.2s, #0
+; CHECK-GI-NEXT: ushll v3.2d, v17.2s, #0
+; CHECK-GI-NEXT: ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT: ushll v7.2d, v19.2s, #0
; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0
; CHECK-GI-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-GI-NEXT: ushll v3.2d, v3.2s, #0
; CHECK-GI-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-GI-NEXT: ushll v5.2d, v6.2s, #0
-; CHECK-GI-NEXT: ushll v6.2d, v16.2s, #0
-; CHECK-GI-NEXT: ushll v7.2d, v18.2s, #0
-; CHECK-GI-NEXT: shl v0.2d, v0.2d, #54
+; CHECK-GI-NEXT: ushll v6.2d, v6.2s, #0
; CHECK-GI-NEXT: shl v1.2d, v1.2d, #54
-; CHECK-GI-NEXT: shl v2.2d, v2.2d, #54
; CHECK-GI-NEXT: shl v3.2d, v3.2d, #54
-; CHECK-GI-NEXT: shl v16.2d, v4.2d, #54
; CHECK-GI-NEXT: shl v5.2d, v5.2d, #54
-; CHECK-GI-NEXT: shl v6.2d, v6.2d, #54
; CHECK-GI-NEXT: shl v7.2d, v7.2d, #54
-; CHECK-GI-NEXT: sshr v4.2d, v0.2d, #54
+; CHECK-GI-NEXT: shl v16.2d, v0.2d, #54
+; CHECK-GI-NEXT: shl v17.2d, v2.2d, #54
+; CHECK-GI-NEXT: shl v18.2d, v4.2d, #54
+; CHECK-GI-NEXT: shl v19.2d, v6.2d, #54
; CHECK-GI-NEXT: sshr v0.2d, v1.2d, #54
-; CHECK-GI-NEXT: sshr v1.2d, v2.2d, #54
-; CHECK-GI-NEXT: sshr v2.2d, v3.2d, #54
-; CHECK-GI-NEXT: sshr v3.2d, v16.2d, #54
-; CHECK-GI-NEXT: sshr v5.2d, v5.2d, #54
-; CHECK-GI-NEXT: sshr v6.2d, v6.2d, #54
-; CHECK-GI-NEXT: sshr v7.2d, v7.2d, #54
+; CHECK-GI-NEXT: sshr v1.2d, v3.2d, #54
+; CHECK-GI-NEXT: sshr v2.2d, v5.2d, #54
+; CHECK-GI-NEXT: sshr v3.2d, v7.2d, #54
+; CHECK-GI-NEXT: sshr v4.2d, v16.2d, #54
+; CHECK-GI-NEXT: sshr v5.2d, v17.2d, #54
+; CHECK-GI-NEXT: sshr v6.2d, v18.2d, #54
+; CHECK-GI-NEXT: sshr v7.2d, v19.2d, #54
; CHECK-GI-NEXT: ret
entry:
%c = sext <16 x i10> %a to <16 x i64>
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index 5f24af82d8342..9827cb3526f99 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -598,8 +598,7 @@ define <1 x i32> @shl_v1i32(<1 x i32> %0, <1 x i32> %1){
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: lsl w8, w8, w9
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%3 = shl <1 x i32> %0, %1
ret <1 x i32> %3
@@ -775,8 +774,7 @@ define <1 x i32> @ashr_v1i32(<1 x i32> %0, <1 x i32> %1){
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: asr w8, w8, w9
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%3 = ashr <1 x i32> %0, %1
ret <1 x i32> %3
@@ -948,8 +946,7 @@ define <1 x i32> @lshr_v1i32(<1 x i32> %0, <1 x i32> %1){
; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
; CHECK-GI-NEXT: lsr w8, w8, w9
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: ret
%3 = lshr <1 x i32> %0, %1
ret <1 x i32> %3
diff --git a/llvm/test/CodeGen/AArch64/shufflevector.ll b/llvm/test/CodeGen/AArch64/shufflevector.ll
index e5c07e0c81e35..59cc400e8ac73 100644
--- a/llvm/test/CodeGen/AArch64/shufflevector.ll
+++ b/llvm/test/CodeGen/AArch64/shufflevector.ll
@@ -217,7 +217,7 @@ define <2 x i1> @shufflevector_v2i1(<2 x i1> %a, <2 x i1> %b){
; CHECK-GI-NEXT: mov v0.b[1], v1.b[1]
; CHECK-GI-NEXT: umov w8, v0.b[0]
; CHECK-GI-NEXT: umov w9, v0.b[1]
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
@@ -395,21 +395,15 @@ define <3 x ptr> @shufflevector_v3p0(<3 x ptr> %a, <3 x ptr> %b) {
;
; CHECK-GI-LABEL: shufflevector_v3p0:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: fmov x9, d3
-; CHECK-GI-NEXT: mov v0.d[0], x8
-; CHECK-GI-NEXT: mov v2.d[0], x9
; CHECK-GI-NEXT: fmov x8, d1
; CHECK-GI-NEXT: fmov x9, d4
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT: fmov d2, d5
; CHECK-GI-NEXT: mov v0.d[1], x8
-; CHECK-GI-NEXT: mov v2.d[1], x9
-; CHECK-GI-NEXT: fmov x8, d5
-; CHECK-GI-NEXT: mov v1.d[0], x8
-; CHECK-GI-NEXT: ext v0.16b, v0.16b, v2.16b, #8
-; CHECK-GI-NEXT: fmov x10, d1
-; CHECK-GI-NEXT: mov d2, v0.d[1]
-; CHECK-GI-NEXT: fmov d1, d2
-; CHECK-GI-NEXT: fmov d2, x10
+; CHECK-GI-NEXT: mov v3.d[1], x9
+; CHECK-GI-NEXT: ext v0.16b, v0.16b, v3.16b, #8
+; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: ret
%c = shufflevector <3 x ptr> %a, <3 x ptr> %b, <3 x i32> <i32 1, i32 3, i32 5>
ret <3 x ptr> %c
@@ -449,7 +443,7 @@ define <2 x i1> @shufflevector_v2i1_zeroes(<2 x i1> %a, <2 x i1> %b){
; CHECK-GI-NEXT: dup v0.8b, v0.b[0]
; CHECK-GI-NEXT: umov w8, v0.b[0]
; CHECK-GI-NEXT: umov w9, v0.b[1]
-; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/trunc.ll b/llvm/test/CodeGen/AArch64/trunc.ll
index d510dc83cfd86..18a35b85eb414 100644
--- a/llvm/test/CodeGen/AArch64/trunc.ll
+++ b/llvm/test/CodeGen/AArch64/trunc.ll
@@ -297,10 +297,10 @@ define <3 x i32> @trunc_v3i32_v3i64(<3 x i64> %a) {
; CHECK-GI-LABEL: trunc_v3i32_v3i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: fmov x8, d1
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: fmov x8, d2
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -867,17 +867,11 @@ entry:
}
define <2 x i64> @trunc_v2i64_v2i128(<2 x i128> %a) {
-; CHECK-SD-LABEL: trunc_v2i64_v2i128:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov d0, x0
-; CHECK-SD-NEXT: mov v0.d[1], x2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: trunc_v2i64_v2i128:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.d[0], x0
-; CHECK-GI-NEXT: mov v0.d[1], x2
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: trunc_v2i64_v2i128:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov d0, x0
+; CHECK-NEXT: mov v0.d[1], x2
+; CHECK-NEXT: ret
entry:
%c = trunc <2 x i128> %a to <2 x i64>
ret <2 x i64> %c
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index 4e2ca082e28b5..77483ebb2235c 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -598,7 +598,7 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
; CHECK-GI-NEXT: mvn.8b v0, v0
; CHECK-GI-NEXT: umov.b w8, v0[0]
; CHECK-GI-NEXT: umov.b w9, v0[1]
-; CHECK-GI-NEXT: mov.s v1[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: umov.b w8, v0[2]
; CHECK-GI-NEXT: mov.s v1[1], w9
; CHECK-GI-NEXT: umov.b w9, v0[3]
@@ -858,37 +858,37 @@ define i6 @no_combine_illegal_num_elements(<6 x i32> %vec) {
; CHECK-GI: ; %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: mov.s v0[0], w0
-; CHECK-GI-NEXT: mov.s v1[0], w4
-; CHECK-GI-NEXT: mov.s v2[0], wzr
-; CHECK-GI-NEXT: mov.s v0[1], w1
-; CHECK-GI-NEXT: mov.s v1[1], w5
-; CHECK-GI-NEXT: mov.s v2[1], wzr
-; CHECK-GI-NEXT: mov.s v0[2], w2
-; CHECK-GI-NEXT: cmeq.4s v1, v1, v2
-; CHECK-GI-NEXT: mvn.16b v1, v1
-; CHECK-GI-NEXT: mov.s v0[3], w3
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
-; CHECK-GI-NEXT: mov.s w10, v0[3]
-; CHECK-GI-NEXT: mov.h v0[1], w8
+; CHECK-GI-NEXT: fmov s1, w0
+; CHECK-GI-NEXT: fmov s0, wzr
+; CHECK-GI-NEXT: fmov s2, w4
+; CHECK-GI-NEXT: mov.s v1[1], w1
+; CHECK-GI-NEXT: mov.s v2[1], w5
+; CHECK-GI-NEXT: mov.s v0[1], wzr
+; CHECK-GI-NEXT: mov.s v1[2], w2
+; CHECK-GI-NEXT: cmeq.4s v0, v2, v0
+; CHECK-GI-NEXT: mvn.16b v0, v0
+; CHECK-GI-NEXT: mov.s v1[3], w3
+; CHECK-GI-NEXT: cmtst.4s v1, v1, v1
; CHECK-GI-NEXT: mov.s w8, v1[1]
-; CHECK-GI-NEXT: mov.h v0[2], w9
-; CHECK-GI-NEXT: mov.h v0[3], w10
-; CHECK-GI-NEXT: mov.h v0[4], v1[0]
-; CHECK-GI-NEXT: mov.h v0[5], w8
-; CHECK-GI-NEXT: umov.h w8, v0[1]
-; CHECK-GI-NEXT: umov.h w9, v0[0]
-; CHECK-GI-NEXT: umov.h w10, v0[2]
-; CHECK-GI-NEXT: umov.h w11, v0[3]
+; CHECK-GI-NEXT: mov.s w9, v1[2]
+; CHECK-GI-NEXT: mov.s w10, v1[3]
+; CHECK-GI-NEXT: mov.h v1[1], w8
+; CHECK-GI-NEXT: mov.s w8, v0[1]
+; CHECK-GI-NEXT: mov.h v1[2], w9
+; CHECK-GI-NEXT: mov.h v1[3], w10
+; CHECK-GI-NEXT: mov.h v1[4], v0[0]
+; CHECK-GI-NEXT: mov.h v1[5], w8
+; CHECK-GI-NEXT: umov.h w8, v1[1]
+; CHECK-GI-NEXT: umov.h w9, v1[0]
+; CHECK-GI-NEXT: umov.h w10, v1[2]
+; CHECK-GI-NEXT: umov.h w11, v1[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.h w10, v0[4]
+; CHECK-GI-NEXT: umov.h w10, v1[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.h w11, v0[5]
+; CHECK-GI-NEXT: umov.h w11, v1[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index fd24282366282..290a473df6bfe 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -2018,29 +2018,33 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-BASE-LABEL: test_udot_v24i8:
; CHECK-GI-BASE: // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT: mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT: ldr q1, [x0]
-; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT: ldr q3, [x1]
-; CHECK-GI-BASE-NEXT: ldr d4, [x1, #16]
-; CHECK-GI-BASE-NEXT: ushll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v2.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT: ushll v6.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT: fmov s0, wzr
+; CHECK-GI-BASE-NEXT: fmov s1, wzr
+; CHECK-GI-BASE-NEXT: ldr q2, [x0]
+; CHECK-GI-BASE-NEXT: ldr d3, [x0, #16]
+; CHECK-GI-BASE-NEXT: ldr q4, [x1]
+; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16]
+; CHECK-GI-BASE-NEXT: ushll v6.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v2.8h, v2.16b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr
-; CHECK-GI-BASE-NEXT: umull v7.4s, v6.4h, v5.4h
-; CHECK-GI-BASE-NEXT: umull v16.4s, v3.4h, v1.4h
-; CHECK-GI-BASE-NEXT: umull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT: ushll v7.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT: ushll v5.8h, v5.8b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT: umlal2 v7.4s, v6.8h, v5.8h
-; CHECK-GI-BASE-NEXT: umlal2 v16.4s, v3.8h, v1.8h
-; CHECK-GI-BASE-NEXT: umlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT: umull v16.4s, v7.4h, v6.4h
+; CHECK-GI-BASE-NEXT: umull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT: umull v18.4s, v5.4h, v3.4h
; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT: add v1.4s, v7.4s, v16.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT: umlal2 v16.4s, v7.8h, v6.8h
+; CHECK-GI-BASE-NEXT: umlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT: umlal2 v18.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v16.4s, v17.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v18.4s, v0.4s
; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-BASE-NEXT: addv s0, v0.4s
; CHECK-GI-BASE-NEXT: fmov w0, s0
@@ -2114,45 +2118,58 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-BASE-LABEL: test_udot_v48i8:
; CHECK-GI-BASE: // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT: mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT: ldp q1, q5, [x1]
-; CHECK-GI-BASE-NEXT: ldp q2, q3, [x0]
-; CHECK-GI-BASE-NEXT: ldr q4, [x0, #32]
-; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32]
-; CHECK-GI-BASE-NEXT: ushll v7.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT: fmov s0, wzr
+; CHECK-GI-BASE-NEXT: fmov s2, wzr
+; CHECK-GI-BASE-NEXT: ldr q16, [x0, #32]
+; CHECK-GI-BASE-NEXT: fmov s1, wzr
+; CHECK-GI-BASE-NEXT: fmov s3, wzr
+; CHECK-GI-BASE-NEXT: ldr q19, [x1, #32]
+; CHECK-GI-BASE-NEXT: ldp q5, q7, [x1]
+; CHECK-GI-BASE-NEXT: ushll v23.8h, v16.8b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT: mov v2.s[1], wzr
+; CHECK-GI-BASE-NEXT: ushll v20.8h, v19.8b, #0
+; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[1], wzr
+; CHECK-GI-BASE-NEXT: ushll2 v19.8h, v19.16b, #0
+; CHECK-GI-BASE-NEXT: ldp q18, q17, [x0]
+; CHECK-GI-BASE-NEXT: ushll v4.8h, v5.8b, #0
; CHECK-GI-BASE-NEXT: ushll2 v5.8h, v5.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v17.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v18.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v19.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT: ushll v20.8h, v4.8b, #0
-; CHECK-GI-BASE-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT: ushll v6.8h, v7.8b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v7.8h, v7.16b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v16.8h, v16.16b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT: umull v21.4s, v7.4h, v18.4h
-; CHECK-GI-BASE-NEXT: umull v22.4s, v1.4h, v2.4h
-; CHECK-GI-BASE-NEXT: umull v23.4s, v16.4h, v19.4h
-; CHECK-GI-BASE-NEXT: umull v24.4s, v5.4h, v3.4h
-; CHECK-GI-BASE-NEXT: umull v25.4s, v17.4h, v20.4h
-; CHECK-GI-BASE-NEXT: umull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT: mov v2.s[2], wzr
+; CHECK-GI-BASE-NEXT: ushll v21.8h, v18.8b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v18.8h, v18.16b, #0
+; CHECK-GI-BASE-NEXT: ushll v22.8h, v17.8b, #0
+; CHECK-GI-BASE-NEXT: ushll2 v17.8h, v17.16b, #0
+; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[2], wzr
+; CHECK-GI-BASE-NEXT: umull v28.4s, v20.4h, v23.4h
+; CHECK-GI-BASE-NEXT: umull v29.4s, v19.4h, v16.4h
+; CHECK-GI-BASE-NEXT: umull v24.4s, v4.4h, v21.4h
+; CHECK-GI-BASE-NEXT: umull v25.4s, v5.4h, v18.4h
+; CHECK-GI-BASE-NEXT: umull v26.4s, v6.4h, v22.4h
+; CHECK-GI-BASE-NEXT: umull v27.4s, v7.4h, v17.4h
; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT: umlal2 v21.4s, v7.8h, v18.8h
-; CHECK-GI-BASE-NEXT: umlal2 v22.4s, v1.8h, v2.8h
-; CHECK-GI-BASE-NEXT: umlal2 v23.4s, v16.8h, v19.8h
-; CHECK-GI-BASE-NEXT: umlal2 v24.4s, v5.8h, v3.8h
-; CHECK-GI-BASE-NEXT: umlal2 v25.4s, v17.8h, v20.8h
-; CHECK-GI-BASE-NEXT: umlal2 v26.4s, v6.8h, v4.8h
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v1.4s, v21.4s, v22.4s
-; CHECK-GI-BASE-NEXT: add v2.4s, v23.4s, v24.4s
-; CHECK-GI-BASE-NEXT: add v3.4s, v25.4s, v26.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT: mov v2.s[3], wzr
+; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[3], wzr
+; CHECK-GI-BASE-NEXT: umlal2 v28.4s, v20.8h, v23.8h
+; CHECK-GI-BASE-NEXT: umlal2 v29.4s, v19.8h, v16.8h
+; CHECK-GI-BASE-NEXT: umlal2 v24.4s, v4.8h, v21.8h
+; CHECK-GI-BASE-NEXT: umlal2 v25.4s, v5.8h, v18.8h
+; CHECK-GI-BASE-NEXT: umlal2 v26.4s, v6.8h, v22.8h
+; CHECK-GI-BASE-NEXT: umlal2 v27.4s, v7.8h, v17.8h
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-BASE-NEXT: add v4.4s, v28.4s, v29.4s
+; CHECK-GI-BASE-NEXT: add v2.4s, v24.4s, v25.4s
+; CHECK-GI-BASE-NEXT: add v3.4s, v26.4s, v27.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v2.4s, v3.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v0.4s
; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-BASE-NEXT: addv s0, v0.4s
; CHECK-GI-BASE-NEXT: fmov w0, s0
@@ -2160,15 +2177,15 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-DOT-LABEL: test_udot_v48i8:
; CHECK-GI-DOT: // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT: mov v0.s[0], wzr
+; CHECK-GI-DOT-NEXT: fmov s0, wzr
; CHECK-GI-DOT-NEXT: movi v1.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: ldr q7, [x0, #32]
; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: movi v3.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: ldr q17, [x1, #32]
; CHECK-GI-DOT-NEXT: ldp q4, q5, [x0]
-; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]
; CHECK-GI-DOT-NEXT: mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]
; CHECK-GI-DOT-NEXT: udot v2.4s, v17.16b, v7.16b
; CHECK-GI-DOT-NEXT: udot v1.4s, v6.16b, v4.16b
; CHECK-GI-DOT-NEXT: udot v3.4s, v16.16b, v5.16b
@@ -2319,29 +2336,33 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-BASE-LABEL: test_sdot_v24i8:
; CHECK-GI-BASE: // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT: mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT: ldr q1, [x0]
-; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16]
-; CHECK-GI-BASE-NEXT: ldr q3, [x1]
-; CHECK-GI-BASE-NEXT: ldr d4, [x1, #16]
-; CHECK-GI-BASE-NEXT: sshll v5.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v2.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT: sshll v6.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v4.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT: fmov s0, wzr
+; CHECK-GI-BASE-NEXT: fmov s1, wzr
+; CHECK-GI-BASE-NEXT: ldr q2, [x0]
+; CHECK-GI-BASE-NEXT: ldr d3, [x0, #16]
+; CHECK-GI-BASE-NEXT: ldr q4, [x1]
+; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16]
+; CHECK-GI-BASE-NEXT: sshll v6.8h, v2.8b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v2.8h, v2.16b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr
-; CHECK-GI-BASE-NEXT: smull v7.4s, v6.4h, v5.4h
-; CHECK-GI-BASE-NEXT: smull v16.4s, v3.4h, v1.4h
-; CHECK-GI-BASE-NEXT: smull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-GI-BASE-NEXT: sshll v7.8h, v4.8b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT: sshll v5.8h, v5.8b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT: smlal2 v7.4s, v6.8h, v5.8h
-; CHECK-GI-BASE-NEXT: smlal2 v16.4s, v3.8h, v1.8h
-; CHECK-GI-BASE-NEXT: smlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT: smull v16.4s, v7.4h, v6.4h
+; CHECK-GI-BASE-NEXT: smull v17.4s, v4.4h, v2.4h
+; CHECK-GI-BASE-NEXT: smull v18.4s, v5.4h, v3.4h
; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT: add v1.4s, v7.4s, v16.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v17.4s, v0.4s
+; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT: smlal2 v16.4s, v7.8h, v6.8h
+; CHECK-GI-BASE-NEXT: smlal2 v17.4s, v4.8h, v2.8h
+; CHECK-GI-BASE-NEXT: smlal2 v18.4s, v5.8h, v3.8h
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v16.4s, v17.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v18.4s, v0.4s
; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-BASE-NEXT: addv s0, v0.4s
; CHECK-GI-BASE-NEXT: fmov w0, s0
@@ -2415,45 +2436,58 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-BASE-LABEL: test_sdot_v48i8:
; CHECK-GI-BASE: // %bb.0: // %entry
-; CHECK-GI-BASE-NEXT: mov v0.s[0], wzr
-; CHECK-GI-BASE-NEXT: ldp q1, q5, [x1]
-; CHECK-GI-BASE-NEXT: ldp q2, q3, [x0]
-; CHECK-GI-BASE-NEXT: ldr q4, [x0, #32]
-; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32]
-; CHECK-GI-BASE-NEXT: sshll v7.8h, v1.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v16.8h, v5.8b, #0
+; CHECK-GI-BASE-NEXT: fmov s0, wzr
+; CHECK-GI-BASE-NEXT: fmov s2, wzr
+; CHECK-GI-BASE-NEXT: ldr q16, [x0, #32]
+; CHECK-GI-BASE-NEXT: fmov s1, wzr
+; CHECK-GI-BASE-NEXT: fmov s3, wzr
+; CHECK-GI-BASE-NEXT: ldr q19, [x1, #32]
+; CHECK-GI-BASE-NEXT: ldp q5, q7, [x1]
+; CHECK-GI-BASE-NEXT: sshll v23.8h, v16.8b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr
+; CHECK-GI-BASE-NEXT: mov v2.s[1], wzr
+; CHECK-GI-BASE-NEXT: sshll v20.8h, v19.8b, #0
+; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[1], wzr
+; CHECK-GI-BASE-NEXT: sshll2 v19.8h, v19.16b, #0
+; CHECK-GI-BASE-NEXT: ldp q18, q17, [x0]
+; CHECK-GI-BASE-NEXT: sshll v4.8h, v5.8b, #0
; CHECK-GI-BASE-NEXT: sshll2 v5.8h, v5.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v17.8h, v6.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v6.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v18.8h, v2.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v19.8h, v3.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v3.8h, v3.16b, #0
-; CHECK-GI-BASE-NEXT: sshll v20.8h, v4.8b, #0
-; CHECK-GI-BASE-NEXT: sshll2 v4.8h, v4.16b, #0
+; CHECK-GI-BASE-NEXT: sshll v6.8h, v7.8b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v7.8h, v7.16b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v16.8h, v16.16b, #0
; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr
-; CHECK-GI-BASE-NEXT: smull v21.4s, v7.4h, v18.4h
-; CHECK-GI-BASE-NEXT: smull v22.4s, v1.4h, v2.4h
-; CHECK-GI-BASE-NEXT: smull v23.4s, v16.4h, v19.4h
-; CHECK-GI-BASE-NEXT: smull v24.4s, v5.4h, v3.4h
-; CHECK-GI-BASE-NEXT: smull v25.4s, v17.4h, v20.4h
-; CHECK-GI-BASE-NEXT: smull v26.4s, v6.4h, v4.4h
+; CHECK-GI-BASE-NEXT: mov v2.s[2], wzr
+; CHECK-GI-BASE-NEXT: sshll v21.8h, v18.8b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v18.8h, v18.16b, #0
+; CHECK-GI-BASE-NEXT: sshll v22.8h, v17.8b, #0
+; CHECK-GI-BASE-NEXT: sshll2 v17.8h, v17.16b, #0
+; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[2], wzr
+; CHECK-GI-BASE-NEXT: smull v28.4s, v20.4h, v23.4h
+; CHECK-GI-BASE-NEXT: smull v29.4s, v19.4h, v16.4h
+; CHECK-GI-BASE-NEXT: smull v24.4s, v4.4h, v21.4h
+; CHECK-GI-BASE-NEXT: smull v25.4s, v5.4h, v18.4h
+; CHECK-GI-BASE-NEXT: smull v26.4s, v6.4h, v22.4h
+; CHECK-GI-BASE-NEXT: smull v27.4s, v7.4h, v17.4h
; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr
-; CHECK-GI-BASE-NEXT: smlal2 v21.4s, v7.8h, v18.8h
-; CHECK-GI-BASE-NEXT: smlal2 v22.4s, v1.8h, v2.8h
-; CHECK-GI-BASE-NEXT: smlal2 v23.4s, v16.8h, v19.8h
-; CHECK-GI-BASE-NEXT: smlal2 v24.4s, v5.8h, v3.8h
-; CHECK-GI-BASE-NEXT: smlal2 v25.4s, v17.8h, v20.8h
-; CHECK-GI-BASE-NEXT: smlal2 v26.4s, v6.8h, v4.8h
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v1.4s, v21.4s, v22.4s
-; CHECK-GI-BASE-NEXT: add v2.4s, v23.4s, v24.4s
-; CHECK-GI-BASE-NEXT: add v3.4s, v25.4s, v26.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v0.4s
-; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-GI-BASE-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-BASE-NEXT: mov v2.s[3], wzr
+; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr
+; CHECK-GI-BASE-NEXT: mov v3.s[3], wzr
+; CHECK-GI-BASE-NEXT: smlal2 v28.4s, v20.8h, v23.8h
+; CHECK-GI-BASE-NEXT: smlal2 v29.4s, v19.8h, v16.8h
+; CHECK-GI-BASE-NEXT: smlal2 v24.4s, v4.8h, v21.8h
+; CHECK-GI-BASE-NEXT: smlal2 v25.4s, v5.8h, v18.8h
+; CHECK-GI-BASE-NEXT: smlal2 v26.4s, v6.8h, v22.8h
+; CHECK-GI-BASE-NEXT: smlal2 v27.4s, v7.8h, v17.8h
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-GI-BASE-NEXT: add v4.4s, v28.4s, v29.4s
+; CHECK-GI-BASE-NEXT: add v2.4s, v24.4s, v25.4s
+; CHECK-GI-BASE-NEXT: add v3.4s, v26.4s, v27.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-BASE-NEXT: add v1.4s, v2.4s, v3.4s
+; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v0.4s
; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-GI-BASE-NEXT: addv s0, v0.4s
; CHECK-GI-BASE-NEXT: fmov w0, s0
@@ -2461,15 +2495,15 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {
;
; CHECK-GI-DOT-LABEL: test_sdot_v48i8:
; CHECK-GI-DOT: // %bb.0: // %entry
-; CHECK-GI-DOT-NEXT: mov v0.s[0], wzr
+; CHECK-GI-DOT-NEXT: fmov s0, wzr
; CHECK-GI-DOT-NEXT: movi v1.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: ldr q7, [x0, #32]
; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: movi v3.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: ldr q17, [x1, #32]
; CHECK-GI-DOT-NEXT: ldp q4, q5, [x0]
-; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]
; CHECK-GI-DOT-NEXT: mov v0.s[1], wzr
+; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]
; CHECK-GI-DOT-NEXT: sdot v2.4s, v17.16b, v7.16b
; CHECK-GI-DOT-NEXT: sdot v1.4s, v6.16b, v4.16b
; CHECK-GI-DOT-NEXT: sdot v3.4s, v16.16b, v5.16b
diff --git a/llvm/test/CodeGen/AArch64/xtn.ll b/llvm/test/CodeGen/AArch64/xtn.ll
index 8a4d6b8c7b789..3ae829245372f 100644
--- a/llvm/test/CodeGen/AArch64/xtn.ll
+++ b/llvm/test/CodeGen/AArch64/xtn.ll
@@ -127,19 +127,12 @@ entry:
}
define <2 x i8> @xtn_v2i128_v2i8(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i8:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.s[1], w2
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i8:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v0.s[1], w2
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: xtn_v2i128_v2i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: mov v0.s[1], w2
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
entry:
%arg1 = trunc <2 x i128> %a to <2 x i8>
ret <2 x i8> %arg1
@@ -165,19 +158,12 @@ entry:
}
define <2 x i16> @xtn_v2i128_v2i16(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i16:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.s[1], w2
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i16:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v0.s[1], w2
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: xtn_v2i128_v2i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: mov v0.s[1], w2
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
entry:
%arg1 = trunc <2 x i128> %a to <2 x i16>
ret <2 x i16> %arg1
@@ -194,36 +180,23 @@ entry:
}
define <2 x i32> @xtn_v2i128_v2i32(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i32:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.s[1], w2
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i32:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v0.s[1], w2
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: xtn_v2i128_v2i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: mov v0.s[1], w2
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
entry:
%arg1 = trunc <2 x i128> %a to <2 x i32>
ret <2 x i32> %arg1
}
define <2 x i64> @xtn_v2i128_v2i64(<2 x i128> %a) {
-; CHECK-SD-LABEL: xtn_v2i128_v2i64:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fmov d0, x0
-; CHECK-SD-NEXT: mov v0.d[1], x2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: xtn_v2i128_v2i64:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.d[0], x0
-; CHECK-GI-NEXT: mov v0.d[1], x2
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: xtn_v2i128_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov d0, x0
+; CHECK-NEXT: mov v0.d[1], x2
+; CHECK-NEXT: ret
entry:
%arg1 = trunc <2 x i128> %a to <2 x i64>
ret <2 x i64> %arg1
@@ -341,10 +314,10 @@ define <3 x i32> @xtn_v3i64_v3i32(<3 x i64> %a) {
; CHECK-GI-LABEL: xtn_v3i64_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: fmov x8, d0
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: fmov x8, d1
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: fmov x8, d2
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll
index 962486afa3bb8..7078d9b2586a8 100644
--- a/llvm/test/CodeGen/AArch64/zext.ll
+++ b/llvm/test/CodeGen/AArch64/zext.ll
@@ -269,10 +269,10 @@ define <3 x i32> @zext_v3i8_v3i32(<3 x i8> %a) {
; CHECK-GI-LABEL: zext_v3i8_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and w8, w0, #0xff
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: and w8, w1, #0xff
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: and w9, w1, #0xff
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: and w8, w2, #0xff
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -326,7 +326,7 @@ define <3 x i32> @zext_v3i16_v3i32(<3 x i16> %a) {
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT: umov w8, v0.h[0]
; CHECK-GI-NEXT: umov w9, v0.h[1]
-; CHECK-GI-NEXT: mov v1.s[0], w8
+; CHECK-GI-NEXT: fmov s1, w8
; CHECK-GI-NEXT: umov w8, v0.h[2]
; CHECK-GI-NEXT: mov v1.s[1], w9
; CHECK-GI-NEXT: mov v1.s[2], w8
@@ -428,10 +428,10 @@ define <3 x i32> @zext_v3i10_v3i32(<3 x i10> %a) {
; CHECK-GI-LABEL: zext_v3i10_v3i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: and w8, w0, #0x3ff
-; CHECK-GI-NEXT: mov v0.s[0], w8
-; CHECK-GI-NEXT: and w8, w1, #0x3ff
-; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: and w9, w1, #0x3ff
+; CHECK-GI-NEXT: fmov s0, w8
; CHECK-GI-NEXT: and w8, w2, #0x3ff
+; CHECK-GI-NEXT: mov v0.s[1], w9
; CHECK-GI-NEXT: mov v0.s[2], w8
; CHECK-GI-NEXT: ret
entry:
@@ -1169,44 +1169,44 @@ define <16 x i64> @zext_v16i10_v16i64(<16 x i10> %a) {
;
; CHECK-GI-LABEL: zext_v16i10_v16i64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v0.s[0], w0
-; CHECK-GI-NEXT: mov v1.s[0], w2
-; CHECK-GI-NEXT: ldr s3, [sp]
-; CHECK-GI-NEXT: mov v2.s[0], w4
-; CHECK-GI-NEXT: mov v5.s[0], w6
-; CHECK-GI-NEXT: ldr s4, [sp, #8]
-; CHECK-GI-NEXT: ldr s6, [sp, #16]
-; CHECK-GI-NEXT: ldr s7, [sp, #24]
-; CHECK-GI-NEXT: ldr s16, [sp, #32]
-; CHECK-GI-NEXT: ldr s17, [sp, #40]
-; CHECK-GI-NEXT: ldr s18, [sp, #48]
-; CHECK-GI-NEXT: ldr s19, [sp, #56]
-; CHECK-GI-NEXT: mov v0.s[1], w1
-; CHECK-GI-NEXT: mov v1.s[1], w3
-; CHECK-GI-NEXT: mov v3.s[1], v4.s[0]
-; CHECK-GI-NEXT: mov v2.s[1], w5
-; CHECK-GI-NEXT: mov v5.s[1], w7
+; CHECK-GI-NEXT: fmov s16, w0
+; CHECK-GI-NEXT: fmov s17, w2
+; CHECK-GI-NEXT: ldr s1, [sp]
+; CHECK-GI-NEXT: fmov s18, w4
+; CHECK-GI-NEXT: fmov s19, w6
+; CHECK-GI-NEXT: ldr s3, [sp, #8]
+; CHECK-GI-NEXT: ldr s0, [sp, #16]
+; CHECK-GI-NEXT: ldr s4, [sp, #24]
+; CHECK-GI-NEXT: ldr s2, [sp, #32]
+; CHECK-GI-NEXT: ldr s5, [sp, #40]
+; CHECK-GI-NEXT: ldr s6, [sp, #48]
+; CHECK-GI-NEXT: ldr s7, [sp, #56]
+; CHECK-GI-NEXT: mov v16.s[1], w1
+; CHECK-GI-NEXT: mov v17.s[1], w3
+; CHECK-GI-NEXT: mov v18.s[1], w5
+; CHECK-GI-NEXT: mov v19.s[1], w7
+; CHECK-GI-NEXT: mov v1.s[1], v3.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT: mov v2.s[1], v5.s[0]
; CHECK-GI-NEXT: mov v6.s[1], v7.s[0]
-; CHECK-GI-NEXT: mov v16.s[1], v17.s[0]
-; CHECK-GI-NEXT: mov v18.s[1], v19.s[0]
; CHECK-GI-NEXT: adrp x8, .LCPI54_0
-; CHECK-GI-NEXT: ldr q7, [x8, :lo12:.LCPI54_0]
-; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0
-; CHECK-GI-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-GI-NEXT: ushll v4.2d, v5.2s, #0
-; CHECK-GI-NEXT: ushll v5.2d, v3.2s, #0
-; CHECK-GI-NEXT: ushll v6.2d, v6.2s, #0
-; CHECK-GI-NEXT: ushll v16.2d, v16.2s, #0
-; CHECK-GI-NEXT: ushll v17.2d, v18.2s, #0
-; CHECK-GI-NEXT: and v0.16b, v0.16b, v7.16b
-; CHECK-GI-NEXT: and v1.16b, v1.16b, v7.16b
-; CHECK-GI-NEXT: and v2.16b, v2.16b, v7.16b
-; CHECK-GI-NEXT: and v3.16b, v4.16b, v7.16b
-; CHECK-GI-NEXT: and v4.16b, v5.16b, v7.16b
-; CHECK-GI-NEXT: and v5.16b, v6.16b, v7.16b
-; CHECK-GI-NEXT: and v6.16b, v16.16b, v7.16b
-; CHECK-GI-NEXT: and v7.16b, v17.16b, v7.16b
+; CHECK-GI-NEXT: ushll v3.2d, v16.2s, #0
+; CHECK-GI-NEXT: ushll v4.2d, v17.2s, #0
+; CHECK-GI-NEXT: ushll v5.2d, v18.2s, #0
+; CHECK-GI-NEXT: ushll v7.2d, v19.2s, #0
+; CHECK-GI-NEXT: ushll v16.2d, v1.2s, #0
+; CHECK-GI-NEXT: ushll v18.2d, v0.2s, #0
+; CHECK-GI-NEXT: ushll v19.2d, v2.2s, #0
+; CHECK-GI-NEXT: ushll v20.2d, v6.2s, #0
+; CHECK-GI-NEXT: ldr q17, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT: and v0.16b, v3.16b, v17.16b
+; CHECK-GI-NEXT: and v1.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT: and v2.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT: and v3.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT: and v4.16b, v16.16b, v17.16b
+; CHECK-GI-NEXT: and v5.16b, v18.16b, v17.16b
+; CHECK-GI-NEXT: and v6.16b, v19.16b, v17.16b
+; CHECK-GI-NEXT: and v7.16b, v20.16b, v17.16b
; CHECK-GI-NEXT: ret
entry:
%c = zext <16 x i10> %a to <16 x i64>
More information about the llvm-commits
mailing list