[llvm] [AArch64] Extend vecreduce to udot/sdot transformation to support usdot (PR #120094)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 23 03:05:20 PST 2024
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/120094
>From 111f2fd9cfa6f120b668f6f61cbd68464bfd0137 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Mon, 16 Dec 2024 14:41:51 +0000
Subject: [PATCH 1/2] [AArch64] Extend vecreduce to udot/sdot transformation to
support usdot
---
.../Target/AArch64/AArch64ISelLowering.cpp | 34 +-
llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 5442 ++++++++++++++---
2 files changed, 4472 insertions(+), 1004 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 505fae4e840f7e..e455fabfe2e8d9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -18177,16 +18177,38 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
unsigned ExtOpcode = Op0.getOpcode();
SDValue A = Op0;
SDValue B;
+ unsigned DotOpcode;
if (ExtOpcode == ISD::MUL) {
A = Op0.getOperand(0);
B = Op0.getOperand(1);
- if (A.getOpcode() != B.getOpcode() ||
- A.getOperand(0).getValueType() != B.getOperand(0).getValueType())
+ if (A.getOperand(0).getValueType() != B.getOperand(0).getValueType())
return SDValue();
- ExtOpcode = A.getOpcode();
- }
- if (ExtOpcode != ISD::ZERO_EXTEND && ExtOpcode != ISD::SIGN_EXTEND)
+ auto OpCodeA = A.getOpcode();
+ if (OpCodeA != ISD::ZERO_EXTEND && OpCodeA != ISD::SIGN_EXTEND)
+ return SDValue();
+
+ auto OpCodeB = B.getOpcode();
+ if (OpCodeB != ISD::ZERO_EXTEND && OpCodeB != ISD::SIGN_EXTEND)
+ return SDValue();
+
+ if (OpCodeA == OpCodeB) {
+ DotOpcode =
+ OpCodeA == ISD::ZERO_EXTEND ? AArch64ISD::UDOT : AArch64ISD::SDOT;
+ } else {
+ // Check USDOT support support
+ if (!ST->hasMatMulInt8())
+ return SDValue();
+ DotOpcode = AArch64ISD::USDOT;
+ if (OpCodeA == ISD::SIGN_EXTEND)
+ std::swap(A, B);
+ }
+ } else if (ExtOpcode == ISD::ZERO_EXTEND) {
+ DotOpcode = AArch64ISD::UDOT;
+ } else if (ExtOpcode == ISD::SIGN_EXTEND) {
+ DotOpcode = AArch64ISD::SDOT;
+ } else {
return SDValue();
+ }
EVT Op0VT = A.getOperand(0).getValueType();
bool IsValidElementCount = Op0VT.getVectorNumElements() % 8 == 0;
@@ -18212,8 +18234,6 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
NumOfVecReduce = Op0VT.getVectorNumElements() / 8;
TargetType = MVT::v2i32;
}
- auto DotOpcode =
- (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT;
// Handle the case where we need to generate only one Dot operation.
if (NumOfVecReduce == 1) {
SDValue Zeros = DAG.getConstant(0, DL, TargetType);
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index c345c1e50bbbb7..05ac2956da00c7 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1,22 +1,28 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
; CHECK-GI: warning: Instruction selection used fallback path for test_udot_v5i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v5i8_nomla
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8_double
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
@@ -290,6 +296,128 @@ entry:
ret i32 %x
}
+define i32 @test_usdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v4i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr s0, [x0]
+; CHECK-SD-NEXT: ldr s1, [x1]
+; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT: smull v0.4s, v1.4h, v0.4h
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v4i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr w8, [x0]
+; CHECK-GI-NEXT: ldr w9, [x1]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: fmov s2, w9
+; CHECK-GI-NEXT: uxtb w8, w8
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: mov b1, v0.b[1]
+; CHECK-GI-NEXT: mov b3, v0.b[2]
+; CHECK-GI-NEXT: mov b5, v2.b[2]
+; CHECK-GI-NEXT: mov b4, v0.b[3]
+; CHECK-GI-NEXT: mov b0, v2.b[1]
+; CHECK-GI-NEXT: mov b6, v2.b[3]
+; CHECK-GI-NEXT: fmov s2, w9
+; CHECK-GI-NEXT: fmov w10, s1
+; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: fmov w8, s4
+; CHECK-GI-NEXT: fmov w12, s0
+; CHECK-GI-NEXT: uxtb w10, w10
+; CHECK-GI-NEXT: uxtb w11, w11
+; CHECK-GI-NEXT: sxtb w13, w13
+; CHECK-GI-NEXT: uxtb w8, w8
+; CHECK-GI-NEXT: sxtb w12, w12
+; CHECK-GI-NEXT: mov v1.h[1], w10
+; CHECK-GI-NEXT: fmov w10, s6
+; CHECK-GI-NEXT: fmov s0, w11
+; CHECK-GI-NEXT: fmov s3, w13
+; CHECK-GI-NEXT: mov v2.h[1], w12
+; CHECK-GI-NEXT: sxtb w10, w10
+; CHECK-GI-NEXT: mov v0.h[1], w8
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov v3.h[1], w10
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mov v1.d[1], v0.d[0]
+; CHECK-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <4 x i8>, ptr %a
+ %1 = zext <4 x i8> %0 to <4 x i32>
+ %2 = load <4 x i8>, ptr %b
+ %3 = sext <4 x i8> %2 to <4 x i32>
+ %4 = mul nsw <4 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v4i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-SD-NEXT: bic v2.4h, #255, lsl #8
+; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT: shl v3.4s, v3.4s, #24
+; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT: shl v1.4s, v1.4s, #24
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #24
+; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #24
+; CHECK-SD-NEXT: mul v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: mla v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: addv s0, v2.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v4i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: movi v4.2d, #0x0000ff000000ff
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: shl v1.4s, v1.4s, #24
+; CHECK-GI-NEXT: shl v3.4s, v3.4s, #24
+; CHECK-GI-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT: sshr v1.4s, v1.4s, #24
+; CHECK-GI-NEXT: sshr v3.4s, v3.4s, #24
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mul v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <4 x i8> %a to <4 x i32>
+ %bz = sext <4 x i8> %b to <4 x i32>
+ %m1 = mul nuw nsw <4 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)
+ %cz = zext <4 x i8> %c to <4 x i32>
+ %dz = sext <4 x i8> %d to <4 x i32>
+ %m2 = mul nuw nsw <4 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v5i8:
; CHECK: // %bb.0: // %entry
@@ -414,6 +542,65 @@ entry:
ret i32 %x
}
+define i32 @test_usdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_usdot_v5i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h
+; CHECK-NEXT: mov v3.s[0], v2.s[0]
+; CHECK-NEXT: smlal v3.4s, v1.4h, v0.4h
+; CHECK-NEXT: addv s0, v3.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <5 x i8>, ptr %a
+ %1 = zext <5 x i8> %0 to <5 x i32>
+ %2 = load <5 x i8>, ptr %b
+ %3 = sext <5 x i8> %2 to <5 x i32>
+ %4 = mul nsw <5 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
+; CHECK-LABEL: test_usdot_v5i8_double:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h
+; CHECK-NEXT: smull2 v7.4s, v2.8h, v3.8h
+; CHECK-NEXT: mov v6.s[0], v4.s[0]
+; CHECK-NEXT: mov v5.s[0], v7.s[0]
+; CHECK-NEXT: smlal v6.4s, v0.4h, v1.4h
+; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h
+; CHECK-NEXT: add v0.4s, v6.4s, v5.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %az = zext <5 x i8> %a to <5 x i32>
+ %bz = sext <5 x i8> %b to <5 x i32>
+ %m1 = mul nuw nsw <5 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
+ %cz = zext <5 x i8> %c to <5 x i32>
+ %dz = sext <5 x i8> %d to <5 x i32>
+ %m2 = mul nuw nsw <5 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+
define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-LABEL: test_udot_v8i8:
; CHECK: // %bb.0: // %entry
@@ -508,6 +695,77 @@ entry:
ret i32 %2
}
+define i32 @test_usdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
+; CHECK-SD-LABEL: test_usdot_v8i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr d1, [x0]
+; CHECK-SD-NEXT: ldr d2, [x1]
+; CHECK-SD-NEXT: usdot v0.2s, v1.8b, v2.8b
+; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v8i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr d0, [x0]
+; CHECK-GI-NEXT: ldr d1, [x1]
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mla v2.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: addv s0, v2.4s
+; CHECK-GI-NEXT: fmov w0, s0
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <8 x i8>, ptr %a
+ %1 = zext <8 x i8> %0 to <8 x i32>
+ %2 = load <8 x i8>, ptr %b
+ %3 = sext <8 x i8> %2 to <8 x i32>
+ %4 = mul nsw <8 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
+ ret i32 %5
+}
+
+define i32 @test_usdot_swapped_operands_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
+; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr d1, [x0]
+; CHECK-SD-NEXT: ldr d2, [x1]
+; CHECK-SD-NEXT: usdot v0.2s, v2.8b, v1.8b
+; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr d0, [x0]
+; CHECK-GI-NEXT: ldr d1, [x1]
+; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v2.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mla v2.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: addv s0, v2.4s
+; CHECK-GI-NEXT: fmov w0, s0
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <8 x i8>, ptr %a
+ %1 = sext <8 x i8> %0 to <8 x i32>
+ %2 = load <8 x i8>, ptr %b
+ %3 = zext <8 x i8> %2 to <8 x i32>
+ %4 = mul nsw <8 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)
+ ret i32 %5
+}
define i32 @test_udot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v16i8:
@@ -587,6 +845,101 @@ entry:
ret i32 %2
}
+define i32 @test_usdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v16i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q1, [x0]
+; CHECK-SD-NEXT: ldr q2, [x1]
+; CHECK-SD-NEXT: usdot v0.4s, v1.16b, v2.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v16i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: ldr q1, [x1]
+; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v3.8h, #0
+; CHECK-GI-NEXT: sshll2 v7.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v4.4s, v6.4s, v4.4s
+; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s
+; CHECK-GI-NEXT: mla v4.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mla v5.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <16 x i8>, ptr %a
+ %1 = zext <16 x i8> %0 to <16 x i32>
+ %2 = load <16 x i8>, ptr %b
+ %3 = sext <16 x i8> %2 to <16 x i32>
+ %4 = mul nsw <16 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_swapped_operands_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q1, [x0]
+; CHECK-SD-NEXT: ldr q2, [x1]
+; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: ldr q1, [x1]
+; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v6.4s, v3.8h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v4.4s, v6.4s, v4.4s
+; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s
+; CHECK-GI-NEXT: mla v4.4s, v3.4s, v2.4s
+; CHECK-GI-NEXT: mla v5.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: add v0.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <16 x i8>, ptr %a
+ %1 = sext <16 x i8> %0 to <16 x i32>
+ %2 = load <16 x i8>, ptr %b
+ %3 = zext <16 x i8> %2 to <16 x i32>
+ %4 = mul nsw <16 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
; CHECK-SD-LABEL: test_udot_v8i8_double:
@@ -860,19 +1213,253 @@ entry:
ret i32 %x
}
-define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_udot_v24i8:
+
+define i32 @test_usdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v8i8_double:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q2, [x0]
-; CHECK-SD-NEXT: ldr q3, [x1]
-; CHECK-SD-NEXT: ldr d4, [x0, #16]
-; CHECK-SD-NEXT: ldr d5, [x1, #16]
-; CHECK-SD-NEXT: udot v1.2s, v5.8b, v4.8b
-; CHECK-SD-NEXT: udot v0.4s, v3.16b, v2.16b
-; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
-; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v5.2s, v0.8b, v1.8b
+; CHECK-SD-NEXT: usdot v4.2s, v2.8b, v3.8b
+; CHECK-SD-NEXT: add v0.2s, v5.2s, v4.2s
+; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v8i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll2 v4.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v5.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll2 v6.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll2 v7.4s, v3.8h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mul v4.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mul v5.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mla v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mla v5.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: addv s0, v4.4s
+; CHECK-GI-NEXT: addv s1, v5.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <8 x i8> %a to <8 x i32>
+ %bz = sext <8 x i8> %b to <8 x i32>
+ %m1 = mul nuw nsw <8 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
+ %cz = zext <8 x i8> %c to <8 x i32>
+ %dz = sext <8 x i8> %d to <8 x i32>
+ %m2 = mul nuw nsw <8 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_usdot_swapped_operands_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v5.2s, v1.8b, v0.8b
+; CHECK-SD-NEXT: usdot v4.2s, v3.8b, v2.8b
+; CHECK-SD-NEXT: add v0.2s, v5.2s, v4.2s
+; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: sshll2 v4.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v2.8h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v3.8h, #0
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mul v4.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mul v5.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mla v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mla v5.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: addv s0, v4.4s
+; CHECK-GI-NEXT: addv s1, v5.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %az = sext <8 x i8> %a to <8 x i32>
+ %bz = zext <8 x i8> %b to <8 x i32>
+ %m1 = mul nuw nsw <8 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)
+ %cz = sext <8 x i8> %c to <8 x i32>
+ %dz = zext <8 x i8> %d to <8 x i32>
+ %m2 = mul nuw nsw <8 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_usdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v16i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v5.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT: usdot v4.4s, v2.16b, v3.16b
+; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v16i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v5.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v6.8h, v2.8b, #0
+; CHECK-GI-NEXT: sshll v7.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-NEXT: ushll2 v16.4s, v4.8h, #0
+; CHECK-GI-NEXT: ushll2 v17.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v5.8h, #0
+; CHECK-GI-NEXT: sshll2 v19.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll2 v20.4s, v6.8h, #0
+; CHECK-GI-NEXT: sshll2 v21.4s, v7.8h, #0
+; CHECK-GI-NEXT: ushll2 v22.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll2 v23.4s, v3.8h, #0
+; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v18.4s
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: mul v17.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT: mul v18.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v19.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll v7.4s, v7.4h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mla v17.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mla v18.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mla v19.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <16 x i8> %a to <16 x i32>
+ %bz = sext <16 x i8> %b to <16 x i32>
+ %m1 = mul nuw nsw <16 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
+ %cz = zext <16 x i8> %c to <16 x i32>
+ %dz = sext <16 x i8> %d to <16 x i32>
+ %m2 = mul nuw nsw <16 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+
+define i32 @test_usdot_swapped_operands_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v5.4s, v1.16b, v0.16b
+; CHECK-SD-NEXT: usdot v4.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: sshll v4.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: sshll v6.8h, v2.8b, #0
+; CHECK-GI-NEXT: ushll v7.8h, v3.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: ushll2 v3.8h, v3.16b, #0
+; CHECK-GI-NEXT: sshll2 v16.4s, v4.8h, #0
+; CHECK-GI-NEXT: sshll2 v17.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v18.4s, v5.8h, #0
+; CHECK-GI-NEXT: ushll2 v19.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll2 v20.4s, v6.8h, #0
+; CHECK-GI-NEXT: ushll2 v21.4s, v7.8h, #0
+; CHECK-GI-NEXT: sshll2 v22.4s, v2.8h, #0
+; CHECK-GI-NEXT: ushll2 v23.4s, v3.8h, #0
+; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v18.4s
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: mul v17.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT: mul v18.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v19.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v7.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: mla v17.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mla v18.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: mla v19.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %az = sext <16 x i8> %a to <16 x i32>
+ %bz = zext <16 x i8> %b to <16 x i32>
+ %m1 = mul nuw nsw <16 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)
+ %cz = sext <16 x i8> %c to <16 x i32>
+ %dz = zext <16 x i8> %d to <16 x i32>
+ %m2 = mul nuw nsw <16 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_udot_v24i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q2, [x0]
+; CHECK-SD-NEXT: ldr q3, [x1]
+; CHECK-SD-NEXT: ldr d4, [x0, #16]
+; CHECK-SD-NEXT: ldr d5, [x1, #16]
+; CHECK-SD-NEXT: udot v1.2s, v5.8b, v4.8b
+; CHECK-SD-NEXT: udot v0.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w8, s1
; CHECK-SD-NEXT: fmov w9, s0
; CHECK-SD-NEXT: add w8, w9, w8
@@ -1658,91 +2245,661 @@ entry:
ret i32 %x
}
-
-define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v25i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q3, q0, [x1]
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT: ushll v3.4s, v4.4h, #0
-; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT: mov v5.s[0], v3.s[0]
-; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
+define i32 @test_usdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v24i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q2, [x0]
+; CHECK-SD-NEXT: ldr q3, [x1]
+; CHECK-SD-NEXT: ldr d4, [x1, #16]
+; CHECK-SD-NEXT: ldr d5, [x0, #16]
+; CHECK-SD-NEXT: usdot v1.2s, v5.8b, v4.8b
+; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v3.16b
+; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s1
+; CHECK-SD-NEXT: fmov w9, s0
+; CHECK-SD-NEXT: add w8, w9, w8
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v24i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldr q0, [x0]
+; CHECK-GI-NEXT: ldr d1, [x0, #16]
+; CHECK-GI-NEXT: ldr q2, [x1]
+; CHECK-GI-NEXT: ldr d3, [x1, #16]
+; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v5.8h, v2.8b, #0
+; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll v6.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT: ushll v7.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: ushll v17.4s, v1.4h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll v19.4s, v3.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT: mul v6.4s, v16.4s, v6.4s
+; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s
+; CHECK-GI-NEXT: mul v5.4s, v19.4s, v17.4s
+; CHECK-GI-NEXT: mul v2.4s, v2.4s, v7.4s
+; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: addv s3, v6.4s
+; CHECK-GI-NEXT: addv s4, v4.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s5, v5.4s
+; CHECK-GI-NEXT: addv s2, v2.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: fmov w8, s3
+; CHECK-GI-NEXT: fmov w9, s4
+; CHECK-GI-NEXT: fmov w10, s0
+; CHECK-GI-NEXT: fmov w11, s5
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: fmov w11, s1
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w9, w10, w11
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
entry:
- %0 = load <25 x i8>, ptr %a
- %1 = zext <25 x i8> %0 to <25 x i32>
- %2 = load <25 x i8>, ptr %b
- %3 = zext <25 x i8> %2 to <25 x i32>
- %4 = mul nuw nsw <25 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
- %op.extra = add i32 %5, %sum
+ %0 = load <24 x i8>, ptr %a
+ %1 = zext <24 x i8> %0 to <24 x i32>
+ %2 = load <24 x i8>, ptr %b
+ %3 = sext <24 x i8> %2 to <24 x i32>
+ %4 = mul nsw <24 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
-define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v25i8_nomla:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
-; CHECK-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NEXT: ushll v4.8h, v2.8b, #0
-; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h
-; CHECK-NEXT: mov v0.s[0], v3.s[0]
-; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a1
- %1 = zext <25 x i8> %0 to <25 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
- ret i32 %2
-}
-define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v25i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q3, q0, [x1]
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT: sshll v3.4s, v4.4h, #0
-; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT: mov v5.s[0], v3.s[0]
-; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a
- %1 = sext <25 x i8> %0 to <25 x i32>
+define i32 @test_usdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v24i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT: .cfi_offset w29, -16
+; CHECK-SD-NEXT: fmov s0, w0
+; CHECK-SD-NEXT: ldr b1, [sp, #144]
+; CHECK-SD-NEXT: add x10, sp, #152
+; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: add x8, sp, #168
+; CHECK-SD-NEXT: ldr b2, [sp, #272]
+; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10]
+; CHECK-SD-NEXT: add x11, sp, #280
+; CHECK-SD-NEXT: ldr b3, [sp, #80]
+; CHECK-SD-NEXT: mov v0.b[1], w1
+; CHECK-SD-NEXT: ldr b4, [sp, #528]
+; CHECK-SD-NEXT: add x10, sp, #88
+; CHECK-SD-NEXT: ld1 { v2.b }[1], [x11]
+; CHECK-SD-NEXT: add x11, sp, #536
+; CHECK-SD-NEXT: ldr b5, [sp, #336]
+; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
+; CHECK-SD-NEXT: add x10, sp, #344
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT: add x11, sp, #176
+; CHECK-SD-NEXT: ldr b6, [sp, #656]
+; CHECK-SD-NEXT: mov v0.b[2], w2
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
+; CHECK-SD-NEXT: ldr b7, [sp, #464]
+; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
+; CHECK-SD-NEXT: add x12, sp, #664
+; CHECK-SD-NEXT: add x9, sp, #472
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT: add x8, sp, #96
+; CHECK-SD-NEXT: add x10, sp, #184
+; CHECK-SD-NEXT: add x12, sp, #288
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
+; CHECK-SD-NEXT: mov v0.b[3], w3
+; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11]
+; CHECK-SD-NEXT: add x8, sp, #352
+; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
+; CHECK-SD-NEXT: add x13, sp, #544
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #672
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x13]
+; CHECK-SD-NEXT: add x9, sp, #192
+; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #480
+; CHECK-SD-NEXT: mov v0.b[4], w4
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #296
+; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #552
+; CHECK-SD-NEXT: add x12, sp, #200
+; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #360
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #104
+; CHECK-SD-NEXT: add x9, sp, #560
+; CHECK-SD-NEXT: mov v0.b[5], w5
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #368
+; CHECK-SD-NEXT: ld1 { v1.b }[7], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9]
+; CHECK-SD-NEXT: add x13, sp, #208
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT: add x12, sp, #304
+; CHECK-SD-NEXT: add x8, sp, #568
+; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
+; CHECK-SD-NEXT: add x12, sp, #16
+; CHECK-SD-NEXT: add x17, sp, #376
+; CHECK-SD-NEXT: mov v0.b[6], w6
+; CHECK-SD-NEXT: ld1 { v1.b }[8], [x13]
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8]
+; CHECK-SD-NEXT: add x14, sp, #216
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x17]
+; CHECK-SD-NEXT: add x13, sp, #576
+; CHECK-SD-NEXT: add x11, sp, #224
+; CHECK-SD-NEXT: add x10, sp, #232
+; CHECK-SD-NEXT: add x15, sp, #240
+; CHECK-SD-NEXT: ld1 { v1.b }[9], [x14]
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
+; CHECK-SD-NEXT: add x13, sp, #384
+; CHECK-SD-NEXT: mov v0.b[7], w7
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x13]
+; CHECK-SD-NEXT: add x13, sp, #112
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13]
+; CHECK-SD-NEXT: add x13, sp, #32
+; CHECK-SD-NEXT: add x14, sp, #584
+; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11]
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x14]
+; CHECK-SD-NEXT: add x11, sp, #312
+; CHECK-SD-NEXT: add x14, sp, #40
+; CHECK-SD-NEXT: ld1 { v2.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #592
+; CHECK-SD-NEXT: ld1 { v0.b }[8], [x12]
+; CHECK-SD-NEXT: add x12, sp, #24
+; CHECK-SD-NEXT: add x16, sp, #248
+; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10]
+; CHECK-SD-NEXT: ld1 { v4.b }[8], [x11]
+; CHECK-SD-NEXT: add x11, sp, #400
+; CHECK-SD-NEXT: add x9, sp, #256
+; CHECK-SD-NEXT: add x8, sp, #264
+; CHECK-SD-NEXT: add x10, sp, #72
+; CHECK-SD-NEXT: ld1 { v0.b }[9], [x12]
+; CHECK-SD-NEXT: add x12, sp, #392
+; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x12]
+; CHECK-SD-NEXT: add x12, sp, #48
+; CHECK-SD-NEXT: ld1 { v1.b }[12], [x15]
+; CHECK-SD-NEXT: add x15, sp, #120
+; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v0.b }[10], [x13]
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x15]
+; CHECK-SD-NEXT: add x15, sp, #408
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11]
+; CHECK-SD-NEXT: add x13, sp, #56
+; CHECK-SD-NEXT: ld1 { v1.b }[13], [x16]
+; CHECK-SD-NEXT: add x11, sp, #64
+; CHECK-SD-NEXT: add x16, sp, #616
+; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v0.b }[11], [x14]
+; CHECK-SD-NEXT: add x14, sp, #600
+; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x15]
+; CHECK-SD-NEXT: add x15, sp, #608
+; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
+; CHECK-SD-NEXT: add x9, sp, #488
+; CHECK-SD-NEXT: add x14, sp, #320
+; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v2.b }[6], [x14]
+; CHECK-SD-NEXT: ld1 { v4.b }[10], [x15]
+; CHECK-SD-NEXT: add x14, sp, #624
+; CHECK-SD-NEXT: add x9, sp, #688
+; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #432
+; CHECK-SD-NEXT: add x12, sp, #328
+; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
+; CHECK-SD-NEXT: add x13, sp, #416
+; CHECK-SD-NEXT: ld1 { v2.b }[7], [x12]
+; CHECK-SD-NEXT: ld1 { v5.b }[10], [x13]
+; CHECK-SD-NEXT: ld1 { v4.b }[11], [x16]
+; CHECK-SD-NEXT: add x16, sp, #680
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x16]
+; CHECK-SD-NEXT: add x13, sp, #632
+; CHECK-SD-NEXT: add x12, sp, #504
+; CHECK-SD-NEXT: ld1 { v0.b }[14], [x11]
+; CHECK-SD-NEXT: add x11, sp, #424
+; CHECK-SD-NEXT: add x15, sp, #128
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x11]
+; CHECK-SD-NEXT: ld1 { v4.b }[12], [x14]
+; CHECK-SD-NEXT: add x11, sp, #696
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x15]
+; CHECK-SD-NEXT: add x9, sp, #640
+; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
+; CHECK-SD-NEXT: add x10, sp, #496
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT: ld1 { v4.b }[13], [x13]
+; CHECK-SD-NEXT: add x10, sp, #440
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #512
+; CHECK-SD-NEXT: add x8, sp, #136
+; CHECK-SD-NEXT: usdot v17.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x10]
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x12]
+; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
+; CHECK-SD-NEXT: add x9, sp, #448
+; CHECK-SD-NEXT: add x10, sp, #704
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
+; CHECK-SD-NEXT: add x8, sp, #648
+; CHECK-SD-NEXT: add x10, sp, #520
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11]
+; CHECK-SD-NEXT: ld1 { v4.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #456
+; CHECK-SD-NEXT: add x9, sp, #712
+; CHECK-SD-NEXT: usdot v19.2s, v3.8b, v2.8b
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT: addv s0, v17.4s
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
+; CHECK-SD-NEXT: addp v1.2s, v19.2s, v19.2s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: usdot v16.4s, v5.16b, v4.16b
+; CHECK-SD-NEXT: usdot v18.2s, v7.8b, v6.8b
+; CHECK-SD-NEXT: fmov w9, s1
+; CHECK-SD-NEXT: addv s2, v16.4s
+; CHECK-SD-NEXT: addp v3.2s, v18.2s, v18.2s
+; CHECK-SD-NEXT: add w8, w8, w9
+; CHECK-SD-NEXT: fmov w10, s2
+; CHECK-SD-NEXT: fmov w11, s3
+; CHECK-SD-NEXT: add w9, w10, w11
+; CHECK-SD-NEXT: add w0, w8, w9
+; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v24i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: .cfi_offset w29, -16
+; CHECK-GI-NEXT: ldr w9, [sp, #16]
+; CHECK-GI-NEXT: ldr w8, [sp, #24]
+; CHECK-GI-NEXT: fmov s0, w0
+; CHECK-GI-NEXT: ldr w11, [sp, #208]
+; CHECK-GI-NEXT: ldr w13, [sp, #32]
+; CHECK-GI-NEXT: ldr w12, [sp, #144]
+; CHECK-GI-NEXT: fmov s1, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #80]
+; CHECK-GI-NEXT: ldr w10, [sp, #152]
+; CHECK-GI-NEXT: fmov s4, w11
+; CHECK-GI-NEXT: fmov s3, w12
+; CHECK-GI-NEXT: ldr w11, [sp, #160]
+; CHECK-GI-NEXT: fmov s2, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #216]
+; CHECK-GI-NEXT: ldr w14, [sp, #128]
+; CHECK-GI-NEXT: mov v1.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: ldr w12, [sp, #240]
+; CHECK-GI-NEXT: mov v4.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #96]
+; CHECK-GI-NEXT: mov v3.b[1], w10
+; CHECK-GI-NEXT: mov v2.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #40]
+; CHECK-GI-NEXT: ldr w10, [sp, #272]
+; CHECK-GI-NEXT: ldr w15, [sp, #312]
+; CHECK-GI-NEXT: mov v0.b[1], w1
+; CHECK-GI-NEXT: mov v1.b[2], w13
+; CHECK-GI-NEXT: fmov s5, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #48]
+; CHECK-GI-NEXT: mov v3.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #232]
+; CHECK-GI-NEXT: ldr w13, [sp, #72]
+; CHECK-GI-NEXT: mov v2.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #280]
+; CHECK-GI-NEXT: mov v0.b[2], w2
+; CHECK-GI-NEXT: mov v1.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #224]
+; CHECK-GI-NEXT: mov v5.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #168]
+; CHECK-GI-NEXT: mov v4.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: mov v3.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #64]
+; CHECK-GI-NEXT: mov v0.b[3], w3
+; CHECK-GI-NEXT: mov v2.b[3], w8
+; CHECK-GI-NEXT: mov v1.b[4], w10
+; CHECK-GI-NEXT: ldr w8, [sp, #288]
+; CHECK-GI-NEXT: ldr w10, [sp, #56]
+; CHECK-GI-NEXT: mov v4.b[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #112]
+; CHECK-GI-NEXT: mov v5.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #176]
+; CHECK-GI-NEXT: mov v0.b[4], w4
+; CHECK-GI-NEXT: mov v2.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #120]
+; CHECK-GI-NEXT: mov v1.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #296]
+; CHECK-GI-NEXT: mov v3.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #136]
+; CHECK-GI-NEXT: mov v4.b[4], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #248]
+; CHECK-GI-NEXT: mov v5.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #184]
+; CHECK-GI-NEXT: mov v0.b[5], w5
+; CHECK-GI-NEXT: mov v2.b[5], w11
+; CHECK-GI-NEXT: mov v1.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #304]
+; CHECK-GI-NEXT: mov v3.b[5], w10
+; CHECK-GI-NEXT: ldr w11, [sp, #192]
+; CHECK-GI-NEXT: ldr w10, [sp, #200]
+; CHECK-GI-NEXT: mov v4.b[5], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #264]
+; CHECK-GI-NEXT: mov v5.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: mov v0.b[6], w6
+; CHECK-GI-NEXT: mov v2.b[6], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #336]
+; CHECK-GI-NEXT: mov v1.b[7], w13
+; CHECK-GI-NEXT: mov v3.b[6], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #344]
+; CHECK-GI-NEXT: ldr w13, [sp, #320]
+; CHECK-GI-NEXT: fmov s6, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #400]
+; CHECK-GI-NEXT: mov v4.b[6], w9
+; CHECK-GI-NEXT: mov v5.b[5], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #408]
+; CHECK-GI-NEXT: ldr w9, [sp, #328]
+; CHECK-GI-NEXT: fmov s7, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #352]
+; CHECK-GI-NEXT: mov v2.b[7], w8
+; CHECK-GI-NEXT: mov v6.b[1], w11
+; CHECK-GI-NEXT: mov v3.b[7], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #416]
+; CHECK-GI-NEXT: mov v4.b[7], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #472]
+; CHECK-GI-NEXT: ldr w11, [sp, #360]
+; CHECK-GI-NEXT: mov v7.b[1], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #464]
+; CHECK-GI-NEXT: mov v5.b[6], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #536]
+; CHECK-GI-NEXT: ldr w8, [sp, #368]
+; CHECK-GI-NEXT: mov v0.b[7], w7
+; CHECK-GI-NEXT: mov v6.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #528]
+; CHECK-GI-NEXT: fmov s16, w15
+; CHECK-GI-NEXT: ldr w15, [sp, #600]
+; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: fmov s17, w14
+; CHECK-GI-NEXT: mov v7.b[2], w10
+; CHECK-GI-NEXT: ldr w14, [sp, #592]
+; CHECK-GI-NEXT: mov v16.b[1], w12
+; CHECK-GI-NEXT: mov v5.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: fmov s18, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #544]
+; CHECK-GI-NEXT: mov v6.b[3], w11
+; CHECK-GI-NEXT: mov v17.b[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #424]
+; CHECK-GI-NEXT: ldr w11, [sp, #608]
+; CHECK-GI-NEXT: ldr w10, [sp, #376]
+; CHECK-GI-NEXT: ldr w12, [sp, #384]
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: mov v7.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #656]
+; CHECK-GI-NEXT: mov v18.b[1], w15
+; CHECK-GI-NEXT: mov v16.b[2], w9
+; CHECK-GI-NEXT: mov v6.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #616]
+; CHECK-GI-NEXT: fmov s19, w13
+; CHECK-GI-NEXT: mov v17.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #664]
+; CHECK-GI-NEXT: ldr w13, [sp, #432]
+; CHECK-GI-NEXT: ushll v20.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: mov v18.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #488]
+; CHECK-GI-NEXT: sshll v21.4s, v3.4h, #0
+; CHECK-GI-NEXT: mov v19.b[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #552]
+; CHECK-GI-NEXT: mov v7.b[4], w13
+; CHECK-GI-NEXT: mov v16.b[3], w11
+; CHECK-GI-NEXT: ldr w13, [sp, #496]
+; CHECK-GI-NEXT: ldr w11, [sp, #440]
+; CHECK-GI-NEXT: mov v17.b[3], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #672]
+; CHECK-GI-NEXT: mov v6.b[5], w10
+; CHECK-GI-NEXT: mov v18.b[3], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #624]
+; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT: mov v19.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #560]
+; CHECK-GI-NEXT: mov v7.b[5], w11
+; CHECK-GI-NEXT: mov v16.b[4], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #680]
+; CHECK-GI-NEXT: ldr w11, [sp, #504]
+; CHECK-GI-NEXT: mov v17.b[4], w14
+; CHECK-GI-NEXT: mov v6.b[6], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #632]
+; CHECK-GI-NEXT: mov v18.b[4], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #512]
+; CHECK-GI-NEXT: ldr w9, [sp, #392]
+; CHECK-GI-NEXT: mov v19.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #568]
+; CHECK-GI-NEXT: ldr w14, [sp, #584]
+; CHECK-GI-NEXT: mov v16.b[5], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #688]
+; CHECK-GI-NEXT: sshll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT: mov v17.b[5], w13
+; CHECK-GI-NEXT: ldr w15, [sp, #448]
+; CHECK-GI-NEXT: mul v20.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: mov v18.b[5], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #576]
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: mov v19.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #640]
+; CHECK-GI-NEXT: mov v6.b[7], w9
+; CHECK-GI-NEXT: mov v16.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #696]
+; CHECK-GI-NEXT: mov v7.b[6], w15
+; CHECK-GI-NEXT: mov v17.b[6], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #648]
+; CHECK-GI-NEXT: ushll v22.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov v18.b[6], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #704]
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov v19.b[5], w10
+; CHECK-GI-NEXT: sshll v21.4s, v4.4h, #0
+; CHECK-GI-NEXT: sshll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT: ldr w8, [sp, #456]
+; CHECK-GI-NEXT: ldr w13, [sp, #520]
+; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT: mov v17.b[7], w14
+; CHECK-GI-NEXT: ldr w10, [sp, #712]
+; CHECK-GI-NEXT: sshll v5.8h, v5.8b, #0
+; CHECK-GI-NEXT: mov v18.b[7], w12
+; CHECK-GI-NEXT: mul v1.4s, v1.4s, v4.4s
+; CHECK-GI-NEXT: addv s4, v20.4s
+; CHECK-GI-NEXT: mov v19.b[6], w11
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
+; CHECK-GI-NEXT: mov v7.b[7], w8
+; CHECK-GI-NEXT: mov v16.b[7], w13
+; CHECK-GI-NEXT: ushll v23.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
+; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: mul v21.4s, v22.4s, v21.4s
+; CHECK-GI-NEXT: fmov w8, s4
+; CHECK-GI-NEXT: mov v19.b[7], w10
+; CHECK-GI-NEXT: sshll v4.8h, v18.8b, #0
+; CHECK-GI-NEXT: fmov w9, s0
+; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0
+; CHECK-GI-NEXT: sshll v18.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
+; CHECK-GI-NEXT: mul v2.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT: ushll v5.8h, v7.8b, #0
+; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0
+; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
+; CHECK-GI-NEXT: mul v3.4s, v23.4s, v3.4s
+; CHECK-GI-NEXT: sshll2 v22.4s, v4.8h, #0
+; CHECK-GI-NEXT: sshll v16.8h, v19.8b, #0
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v18.4s
+; CHECK-GI-NEXT: addv s18, v21.4s
+; CHECK-GI-NEXT: ushll2 v19.4s, v5.8h, #0
+; CHECK-GI-NEXT: ushll v20.4s, v7.4h, #0
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: mul v6.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll v23.4s, v16.4h, #0
+; CHECK-GI-NEXT: fmov w9, s18
+; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT: mul v17.4s, v19.4s, v22.4s
+; CHECK-GI-NEXT: addv s3, v3.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s2, v2.4s
+; CHECK-GI-NEXT: mul v19.4s, v20.4s, v23.4s
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: addv s1, v6.4s
+; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s
+; CHECK-GI-NEXT: mul v5.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT: fmov w10, s3
+; CHECK-GI-NEXT: addv s3, v17.4s
+; CHECK-GI-NEXT: fmov w11, s0
+; CHECK-GI-NEXT: addv s6, v19.4s
+; CHECK-GI-NEXT: fmov w12, s1
+; CHECK-GI-NEXT: addv s4, v4.4s
+; CHECK-GI-NEXT: addv s5, v5.4s
+; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: fmov w10, s2
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: fmov w12, s3
+; CHECK-GI-NEXT: fmov w13, s6
+; CHECK-GI-NEXT: fmov w14, s4
+; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: add w12, w12, w13
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w10, w11, w14
+; CHECK-GI-NEXT: add w11, w12, w13
+; CHECK-GI-NEXT: add w9, w10, w11
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <24 x i8> %a to <24 x i32>
+ %bz = sext <24 x i8> %b to <24 x i32>
+ %m1 = mul nuw nsw <24 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
+ %cz = zext <24 x i8> %c to <24 x i32>
+ %dz = sext <24 x i8> %d to <24 x i32>
+ %m2 = mul nuw nsw <24 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v25i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q3, q0, [x1]
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b
+; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b
+; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b
+; CHECK-NEXT: ushll v3.4s, v4.4h, #0
+; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
+; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h
+; CHECK-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a
+ %1 = zext <25 x i8> %0 to <25 x i32>
+ %2 = load <25 x i8>, ptr %b
+ %3 = zext <25 x i8> %2 to <25 x i32>
+ %4 = mul nuw nsw <25 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+ %op.extra = add i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v25i8_nomla:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll v4.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
+; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h
+; CHECK-NEXT: mov v0.s[0], v3.s[0]
+; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
+; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a1
+ %1 = zext <25 x i8> %0 to <25 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
+ ret i32 %2
+}
+define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v25i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q3, q0, [x1]
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b
+; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b
+; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b
+; CHECK-NEXT: sshll v3.4s, v4.4h, #0
+; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h
+; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h
+; CHECK-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a
+ %1 = sext <25 x i8> %0 to <25 x i32>
%2 = load <25 x i8>, ptr %b
%3 = sext <25 x i8> %2 to <25 x i32>
%4 = mul nsw <25 x i32> %3, %1
@@ -2109,6 +3266,284 @@ entry:
ret i32 %x
}
+define i32 @test_usdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_usdot_v25i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q0, [x0]
+; CHECK-NEXT: ldp q5, q1, [x1]
+; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0
+; CHECK-NEXT: ushll v6.8h, v2.8b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0
+; CHECK-NEXT: sshll v7.8h, v5.8b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h
+; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h
+; CHECK-NEXT: mov v4.s[0], v3.s[0]
+; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0
+; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h
+; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h
+; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h
+; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h
+; CHECK-NEXT: add v0.4s, v6.4s, v4.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v16.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a
+ %1 = zext <25 x i8> %0 to <25 x i32>
+ %2 = load <25 x i8>, ptr %b
+ %3 = sext <25 x i8> %2 to <25 x i32>
+ %4 = mul nsw <25 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-LABEL: test_usdot_v25i8_double:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ldr b1, [sp, #16]
+; CHECK-NEXT: ldr b0, [sp, #80]
+; CHECK-NEXT: add x11, sp, #24
+; CHECK-NEXT: ldr b3, [sp, #216]
+; CHECK-NEXT: ldr b4, [sp, #152]
+; CHECK-NEXT: add x10, sp, #88
+; CHECK-NEXT: ldr b2, [sp, #280]
+; CHECK-NEXT: ld1 { v1.b }[1], [x11]
+; CHECK-NEXT: add x11, sp, #224
+; CHECK-NEXT: add x12, sp, #160
+; CHECK-NEXT: ldr b6, [sp, #480]
+; CHECK-NEXT: ld1 { v0.b }[1], [x10]
+; CHECK-NEXT: add x10, sp, #288
+; CHECK-NEXT: ld1 { v3.b }[1], [x11]
+; CHECK-NEXT: ld1 { v4.b }[1], [x12]
+; CHECK-NEXT: add x12, sp, #488
+; CHECK-NEXT: ld1 { v2.b }[1], [x10]
+; CHECK-NEXT: add x9, sp, #96
+; CHECK-NEXT: ld1 { v6.b }[1], [x12]
+; CHECK-NEXT: add x12, sp, #32
+; CHECK-NEXT: ld1 { v0.b }[2], [x9]
+; CHECK-NEXT: ld1 { v1.b }[2], [x12]
+; CHECK-NEXT: add x12, sp, #232
+; CHECK-NEXT: add x9, sp, #296
+; CHECK-NEXT: ld1 { v3.b }[2], [x12]
+; CHECK-NEXT: add x12, sp, #168
+; CHECK-NEXT: ld1 { v2.b }[2], [x9]
+; CHECK-NEXT: ld1 { v4.b }[2], [x12]
+; CHECK-NEXT: add x12, sp, #40
+; CHECK-NEXT: add x8, sp, #104
+; CHECK-NEXT: ld1 { v1.b }[3], [x12]
+; CHECK-NEXT: ld1 { v0.b }[3], [x8]
+; CHECK-NEXT: add x8, sp, #304
+; CHECK-NEXT: ld1 { v2.b }[3], [x8]
+; CHECK-NEXT: add x8, sp, #48
+; CHECK-NEXT: add x10, sp, #112
+; CHECK-NEXT: add x14, sp, #56
+; CHECK-NEXT: add x12, sp, #240
+; CHECK-NEXT: add x11, sp, #120
+; CHECK-NEXT: ld1 { v1.b }[4], [x8]
+; CHECK-NEXT: ld1 { v0.b }[4], [x10]
+; CHECK-NEXT: add x8, sp, #312
+; CHECK-NEXT: ld1 { v3.b }[3], [x12]
+; CHECK-NEXT: add x12, sp, #176
+; CHECK-NEXT: ld1 { v2.b }[4], [x8]
+; CHECK-NEXT: ld1 { v4.b }[3], [x12]
+; CHECK-NEXT: add x12, sp, #64
+; CHECK-NEXT: ldr b18, [sp, #552]
+; CHECK-NEXT: ld1 { v1.b }[5], [x14]
+; CHECK-NEXT: add x16, sp, #320
+; CHECK-NEXT: ld1 { v0.b }[5], [x11]
+; CHECK-NEXT: ldr b16, [sp, #352]
+; CHECK-NEXT: add x8, sp, #248
+; CHECK-NEXT: ld1 { v2.b }[5], [x16]
+; CHECK-NEXT: add x16, sp, #360
+; CHECK-NEXT: add x15, sp, #128
+; CHECK-NEXT: ld1 { v3.b }[4], [x8]
+; CHECK-NEXT: ld1 { v1.b }[6], [x12]
+; CHECK-NEXT: add x12, sp, #560
+; CHECK-NEXT: ld1 { v16.b }[1], [x16]
+; CHECK-NEXT: ld1 { v18.b }[1], [x12]
+; CHECK-NEXT: add x10, sp, #72
+; CHECK-NEXT: ld1 { v0.b }[6], [x15]
+; CHECK-NEXT: add x15, sp, #184
+; CHECK-NEXT: add x12, sp, #568
+; CHECK-NEXT: add x13, sp, #328
+; CHECK-NEXT: add x14, sp, #256
+; CHECK-NEXT: ld1 { v4.b }[4], [x15]
+; CHECK-NEXT: ld1 { v1.b }[7], [x10]
+; CHECK-NEXT: add x10, sp, #368
+; CHECK-NEXT: ld1 { v18.b }[2], [x12]
+; CHECK-NEXT: add x9, sp, #136
+; CHECK-NEXT: ld1 { v3.b }[5], [x14]
+; CHECK-NEXT: ld1 { v2.b }[6], [x13]
+; CHECK-NEXT: add x13, sp, #496
+; CHECK-NEXT: ld1 { v16.b }[2], [x10]
+; CHECK-NEXT: fmov s5, w0
+; CHECK-NEXT: ld1 { v0.b }[7], [x9]
+; CHECK-NEXT: add x9, sp, #192
+; CHECK-NEXT: ld1 { v6.b }[2], [x13]
+; CHECK-NEXT: add x10, sp, #576
+; CHECK-NEXT: add x11, sp, #264
+; CHECK-NEXT: ld1 { v4.b }[5], [x9]
+; CHECK-NEXT: add x9, sp, #376
+; CHECK-NEXT: ld1 { v18.b }[3], [x10]
+; CHECK-NEXT: ld1 { v3.b }[6], [x11]
+; CHECK-NEXT: add x11, sp, #504
+; CHECK-NEXT: ld1 { v16.b }[3], [x9]
+; CHECK-NEXT: mov v5.b[1], w1
+; CHECK-NEXT: ldr b7, [sp, #144]
+; CHECK-NEXT: ldr b17, [sp, #344]
+; CHECK-NEXT: add x9, sp, #200
+; CHECK-NEXT: ld1 { v6.b }[3], [x11]
+; CHECK-NEXT: add x10, sp, #584
+; CHECK-NEXT: ld1 { v4.b }[6], [x9]
+; CHECK-NEXT: add x9, sp, #384
+; CHECK-NEXT: ld1 { v18.b }[4], [x10]
+; CHECK-NEXT: ushll v7.8h, v7.8b, #0
+; CHECK-NEXT: sshll v17.8h, v17.8b, #0
+; CHECK-NEXT: add x11, sp, #512
+; CHECK-NEXT: ld1 { v16.b }[4], [x9]
+; CHECK-NEXT: mov v5.b[2], w2
+; CHECK-NEXT: ld1 { v6.b }[4], [x11]
+; CHECK-NEXT: add x11, sp, #592
+; CHECK-NEXT: add x10, sp, #392
+; CHECK-NEXT: ldr b19, [sp, #680]
+; CHECK-NEXT: ld1 { v18.b }[5], [x11]
+; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h
+; CHECK-NEXT: ldr b17, [sp, #416]
+; CHECK-NEXT: ld1 { v16.b }[5], [x10]
+; CHECK-NEXT: add x10, sp, #688
+; CHECK-NEXT: add x9, sp, #424
+; CHECK-NEXT: ld1 { v19.b }[1], [x10]
+; CHECK-NEXT: add x10, sp, #600
+; CHECK-NEXT: ldr b20, [sp, #616]
+; CHECK-NEXT: ld1 { v17.b }[1], [x9]
+; CHECK-NEXT: add x11, sp, #400
+; CHECK-NEXT: ld1 { v18.b }[6], [x10]
+; CHECK-NEXT: add x12, sp, #624
+; CHECK-NEXT: mov v5.b[3], w3
+; CHECK-NEXT: ld1 { v16.b }[6], [x11]
+; CHECK-NEXT: add x11, sp, #696
+; CHECK-NEXT: ld1 { v20.b }[1], [x12]
+; CHECK-NEXT: add x9, sp, #432
+; CHECK-NEXT: ld1 { v19.b }[2], [x11]
+; CHECK-NEXT: add x11, sp, #608
+; CHECK-NEXT: ld1 { v17.b }[2], [x9]
+; CHECK-NEXT: ld1 { v18.b }[7], [x11]
+; CHECK-NEXT: add x11, sp, #632
+; CHECK-NEXT: add x10, sp, #408
+; CHECK-NEXT: ld1 { v20.b }[2], [x11]
+; CHECK-NEXT: mov v5.b[4], w4
+; CHECK-NEXT: ld1 { v16.b }[7], [x10]
+; CHECK-NEXT: add x10, sp, #704
+; CHECK-NEXT: add x12, sp, #440
+; CHECK-NEXT: ld1 { v19.b }[3], [x10]
+; CHECK-NEXT: ld1 { v17.b }[3], [x12]
+; CHECK-NEXT: add x12, sp, #640
+; CHECK-NEXT: ld1 { v20.b }[3], [x12]
+; CHECK-NEXT: sshll v18.8h, v18.8b, #0
+; CHECK-NEXT: add x10, sp, #448
+; CHECK-NEXT: ushll v21.8h, v16.8b, #0
+; CHECK-NEXT: add x11, sp, #712
+; CHECK-NEXT: mov v5.b[5], w5
+; CHECK-NEXT: ld1 { v19.b }[4], [x11]
+; CHECK-NEXT: ld1 { v17.b }[4], [x10]
+; CHECK-NEXT: add x10, sp, #648
+; CHECK-NEXT: add x9, sp, #520
+; CHECK-NEXT: ld1 { v20.b }[4], [x10]
+; CHECK-NEXT: ldr b22, [sp, #544]
+; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h
+; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h
+; CHECK-NEXT: ldr b21, [sp, #744]
+; CHECK-NEXT: add x11, sp, #720
+; CHECK-NEXT: ld1 { v6.b }[5], [x9]
+; CHECK-NEXT: add x9, sp, #456
+; CHECK-NEXT: ld1 { v19.b }[5], [x11]
+; CHECK-NEXT: ld1 { v17.b }[5], [x9]
+; CHECK-NEXT: add x9, sp, #656
+; CHECK-NEXT: mov v5.b[6], w6
+; CHECK-NEXT: ushll v22.8h, v22.8b, #0
+; CHECK-NEXT: sshll v21.8h, v21.8b, #0
+; CHECK-NEXT: ld1 { v20.b }[5], [x9]
+; CHECK-NEXT: add x10, sp, #528
+; CHECK-NEXT: add x11, sp, #728
+; CHECK-NEXT: ld1 { v6.b }[6], [x10]
+; CHECK-NEXT: ld1 { v19.b }[6], [x11]
+; CHECK-NEXT: add x10, sp, #464
+; CHECK-NEXT: add x11, sp, #664
+; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h
+; CHECK-NEXT: movi v22.2d, #0000000000000000
+; CHECK-NEXT: ld1 { v17.b }[6], [x10]
+; CHECK-NEXT: ld1 { v20.b }[6], [x11]
+; CHECK-NEXT: mov v5.b[7], w7
+; CHECK-NEXT: add x9, sp, #536
+; CHECK-NEXT: add x10, sp, #736
+; CHECK-NEXT: add x11, sp, #208
+; CHECK-NEXT: ld1 { v6.b }[7], [x9]
+; CHECK-NEXT: ld1 { v19.b }[7], [x10]
+; CHECK-NEXT: ld1 { v4.b }[7], [x11]
+; CHECK-NEXT: add x9, sp, #472
+; CHECK-NEXT: add x10, sp, #672
+; CHECK-NEXT: add x8, sp, #336
+; CHECK-NEXT: ld1 { v17.b }[7], [x9]
+; CHECK-NEXT: ld1 { v20.b }[7], [x10]
+; CHECK-NEXT: mov v22.s[0], v21.s[0]
+; CHECK-NEXT: movi v21.2d, #0000000000000000
+; CHECK-NEXT: ushll v5.8h, v5.8b, #0
+; CHECK-NEXT: ushll v6.8h, v6.8b, #0
+; CHECK-NEXT: sshll v19.8h, v19.8b, #0
+; CHECK-NEXT: ld1 { v2.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #272
+; CHECK-NEXT: sshll v4.8h, v4.8b, #0
+; CHECK-NEXT: ld1 { v3.b }[7], [x8]
+; CHECK-NEXT: ushll v17.8h, v17.8b, #0
+; CHECK-NEXT: sshll v20.8h, v20.8b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h
+; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h
+; CHECK-NEXT: mov v21.s[0], v7.s[0]
+; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h
+; CHECK-NEXT: sshll v2.8h, v2.8b, #0
+; CHECK-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h
+; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h
+; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h
+; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h
+; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h
+; CHECK-NEXT: add v5.4s, v18.4s, v22.4s
+; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h
+; CHECK-NEXT: add v0.4s, v6.4s, v21.4s
+; CHECK-NEXT: add v2.4s, v5.4s, v16.4s
+; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+entry:
+ %az = zext <25 x i8> %a to <25 x i32>
+ %bz = sext <25 x i8> %b to <25 x i32>
+ %m1 = mul nuw nsw <25 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
+ %cz = zext <25 x i8> %c to <25 x i32>
+ %dz = sext <25 x i8> %d to <25 x i32>
+ %m2 = mul nuw nsw <25 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v32i8:
; CHECK-SD: // %bb.0: // %entry
@@ -2293,14 +3728,210 @@ define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ret
entry:
- %az = sext <32 x i8> %a to <32 x i32>
- %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)
- %cz = sext <32 x i8> %c to <32 x i32>
- %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz)
+ %az = sext <32 x i8> %a to <32 x i32>
+ %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)
+ %cz = sext <32 x i8> %c to <32 x i32>
+ %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_usdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v32i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ldp q2, q3, [x0]
+; CHECK-SD-NEXT: ldp q4, q5, [x1]
+; CHECK-SD-NEXT: usdot v1.4s, v3.16b, v5.16b
+; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v4.16b
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v32i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q1, [x1]
+; CHECK-GI-NEXT: ldp q2, q3, [x0]
+; CHECK-GI-NEXT: sshll v4.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v5.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v6.8h, v2.8b, #0
+; CHECK-GI-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: ushll v7.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll2 v3.8h, v3.16b, #0
+; CHECK-GI-NEXT: sshll2 v16.4s, v4.8h, #0
+; CHECK-GI-NEXT: sshll2 v17.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v5.8h, #0
+; CHECK-GI-NEXT: sshll2 v19.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll2 v20.4s, v6.8h, #0
+; CHECK-GI-NEXT: ushll2 v21.4s, v2.8h, #0
+; CHECK-GI-NEXT: ushll2 v22.4s, v7.8h, #0
+; CHECK-GI-NEXT: ushll2 v23.4s, v3.8h, #0
+; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v20.4s
+; CHECK-GI-NEXT: mul v17.4s, v17.4s, v21.4s
+; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0
+; CHECK-GI-NEXT: mul v18.4s, v18.4s, v22.4s
+; CHECK-GI-NEXT: mul v19.4s, v19.4s, v23.4s
+; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll v7.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: mla v16.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT: mla v17.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: mla v18.4s, v5.4s, v7.4s
+; CHECK-GI-NEXT: mla v19.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <32 x i8>, ptr %a
+ %1 = zext <32 x i8> %0 to <32 x i32>
+ %2 = load <32 x i8>, ptr %b
+ %3 = sext <32 x i8> %2 to <32 x i32>
+ %4 = mul nsw <32 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v32i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v3.16b
+; CHECK-SD-NEXT: usdot v18.4s, v0.16b, v2.16b
+; CHECK-SD-NEXT: usdot v17.4s, v4.16b, v6.16b
+; CHECK-SD-NEXT: usdot v19.4s, v5.16b, v7.16b
+; CHECK-SD-NEXT: add v0.4s, v18.4s, v16.4s
+; CHECK-SD-NEXT: add v1.4s, v17.4s, v19.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v32i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT: .cfi_offset b8, -8
+; CHECK-GI-NEXT: .cfi_offset b9, -16
+; CHECK-GI-NEXT: .cfi_offset b10, -24
+; CHECK-GI-NEXT: .cfi_offset b11, -32
+; CHECK-GI-NEXT: .cfi_offset b12, -40
+; CHECK-GI-NEXT: .cfi_offset b13, -48
+; CHECK-GI-NEXT: .cfi_offset b14, -56
+; CHECK-GI-NEXT: .cfi_offset b15, -64
+; CHECK-GI-NEXT: ushll v16.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: ushll v17.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: sshll v18.8h, v2.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: sshll v19.8h, v3.8b, #0
+; CHECK-GI-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-GI-NEXT: ushll v27.8h, v4.8b, #0
+; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-NEXT: ushll v28.8h, v5.8b, #0
+; CHECK-GI-NEXT: sshll v29.8h, v6.8b, #0
+; CHECK-GI-NEXT: sshll2 v6.8h, v6.16b, #0
+; CHECK-GI-NEXT: ushll2 v5.8h, v5.16b, #0
+; CHECK-GI-NEXT: sshll v30.8h, v7.8b, #0
+; CHECK-GI-NEXT: sshll2 v7.8h, v7.16b, #0
+; CHECK-GI-NEXT: ushll2 v20.4s, v16.8h, #0
+; CHECK-GI-NEXT: ushll2 v21.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v22.4s, v17.8h, #0
+; CHECK-GI-NEXT: ushll2 v23.4s, v1.8h, #0
+; CHECK-GI-NEXT: sshll2 v24.4s, v18.8h, #0
+; CHECK-GI-NEXT: sshll2 v25.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll2 v26.4s, v19.8h, #0
+; CHECK-GI-NEXT: sshll2 v31.4s, v3.8h, #0
+; CHECK-GI-NEXT: ushll2 v8.4s, v27.8h, #0
+; CHECK-GI-NEXT: ushll2 v9.4s, v4.8h, #0
+; CHECK-GI-NEXT: ushll2 v10.4s, v28.8h, #0
+; CHECK-GI-NEXT: sshll2 v11.4s, v29.8h, #0
+; CHECK-GI-NEXT: sshll2 v12.4s, v6.8h, #0
+; CHECK-GI-NEXT: ushll2 v13.4s, v5.8h, #0
+; CHECK-GI-NEXT: sshll2 v14.4s, v30.8h, #0
+; CHECK-GI-NEXT: sshll2 v15.4s, v7.8h, #0
+; CHECK-GI-NEXT: mul v20.4s, v20.4s, v24.4s
+; CHECK-GI-NEXT: mul v21.4s, v21.4s, v25.4s
+; CHECK-GI-NEXT: mul v22.4s, v22.4s, v26.4s
+; CHECK-GI-NEXT: mul v23.4s, v23.4s, v31.4s
+; CHECK-GI-NEXT: mul v24.4s, v8.4s, v11.4s
+; CHECK-GI-NEXT: mul v25.4s, v9.4s, v12.4s
+; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mul v26.4s, v10.4s, v14.4s
+; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mul v31.4s, v13.4s, v15.4s
+; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: sshll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll v19.4s, v19.4h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: ushll v27.4s, v27.4h, #0
+; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll v28.4s, v28.4h, #0
+; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll v29.4s, v29.4h, #0
+; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #0
+; CHECK-GI-NEXT: sshll v30.4s, v30.4h, #0
+; CHECK-GI-NEXT: sshll v7.4s, v7.4h, #0
+; CHECK-GI-NEXT: mla v20.4s, v16.4s, v18.4s
+; CHECK-GI-NEXT: mla v21.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: mla v22.4s, v17.4s, v19.4s
+; CHECK-GI-NEXT: mla v23.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: mla v24.4s, v27.4s, v29.4s
+; CHECK-GI-NEXT: mla v25.4s, v4.4s, v6.4s
+; CHECK-GI-NEXT: mla v26.4s, v28.4s, v30.4s
+; CHECK-GI-NEXT: mla v31.4s, v5.4s, v7.4s
+; CHECK-GI-NEXT: add v0.4s, v20.4s, v21.4s
+; CHECK-GI-NEXT: add v1.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: add v2.4s, v24.4s, v25.4s
+; CHECK-GI-NEXT: add v3.4s, v26.4s, v31.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <32 x i8> %a to <32 x i32>
+ %bz = sext <32 x i8> %b to <32 x i32>
+ %m1 = mul nuw nsw <32 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)
+ %cz = zext <32 x i8> %c to <32 x i32>
+ %dz = sext <32 x i8> %d to <32 x i32>
+ %m2 = mul nuw nsw <32 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
+
define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-LABEL: test_udot_v33i8:
; CHECK: // %bb.0: // %entry
@@ -2866,6 +4497,362 @@ entry:
%x = add i32 %r1, %r2
ret i32 %x
}
+
+define i32 @test_usdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_usdot_v33i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr b0, [x0, #32]
+; CHECK-NEXT: ldr b1, [x1, #32]
+; CHECK-NEXT: ldp q2, q4, [x0]
+; CHECK-NEXT: ldp q3, q6, [x1]
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll v5.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v16.8h, v4.8b, #0
+; CHECK-NEXT: sshll v7.8h, v3.8b, #0
+; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sshll v19.8h, v6.8b, #0
+; CHECK-NEXT: sshll2 v6.8h, v6.16b, #0
+; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h
+; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h
+; CHECK-NEXT: smlal2 v18.4s, v6.8h, v4.8h
+; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h
+; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h
+; CHECK-NEXT: smlal v0.4s, v6.4h, v4.4h
+; CHECK-NEXT: add v2.4s, v17.4s, v18.4s
+; CHECK-NEXT: smlal v1.4s, v19.4h, v16.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <33 x i8>, ptr %a
+ %1 = zext <33 x i8> %0 to <33 x i32>
+ %2 = load <33 x i8>, ptr %b
+ %3 = sext <33 x i8> %2 to <33 x i32>
+ %4 = mul nsw <33 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
+; CHECK-LABEL: test_usdot_v33i8_double:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: fmov s4, w0
+; CHECK-NEXT: ldr b0, [sp, #144]
+; CHECK-NEXT: add x8, sp, #152
+; CHECK-NEXT: ldr b1, [sp, #80]
+; CHECK-NEXT: add x10, sp, #88
+; CHECK-NEXT: ldr b6, [sp, #16]
+; CHECK-NEXT: ld1 { v0.b }[1], [x8]
+; CHECK-NEXT: ldr b2, [sp, #408]
+; CHECK-NEXT: add x9, sp, #160
+; CHECK-NEXT: mov v4.b[1], w1
+; CHECK-NEXT: ld1 { v1.b }[1], [x10]
+; CHECK-NEXT: add x10, sp, #24
+; CHECK-NEXT: ld1 { v6.b }[1], [x10]
+; CHECK-NEXT: add x10, sp, #416
+; CHECK-NEXT: add x8, sp, #168
+; CHECK-NEXT: ld1 { v0.b }[2], [x9]
+; CHECK-NEXT: add x9, sp, #96
+; CHECK-NEXT: ld1 { v2.b }[1], [x10]
+; CHECK-NEXT: ld1 { v1.b }[2], [x9]
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: add x10, sp, #424
+; CHECK-NEXT: mov v4.b[2], w2
+; CHECK-NEXT: ld1 { v6.b }[2], [x9]
+; CHECK-NEXT: add x12, sp, #176
+; CHECK-NEXT: ld1 { v2.b }[2], [x10]
+; CHECK-NEXT: add x10, sp, #104
+; CHECK-NEXT: ld1 { v0.b }[3], [x8]
+; CHECK-NEXT: ld1 { v1.b }[3], [x10]
+; CHECK-NEXT: add x10, sp, #40
+; CHECK-NEXT: add x13, sp, #112
+; CHECK-NEXT: ld1 { v6.b }[3], [x10]
+; CHECK-NEXT: add x10, sp, #432
+; CHECK-NEXT: add x11, sp, #184
+; CHECK-NEXT: mov v4.b[3], w3
+; CHECK-NEXT: ld1 { v0.b }[4], [x12]
+; CHECK-NEXT: add x12, sp, #48
+; CHECK-NEXT: ld1 { v2.b }[3], [x10]
+; CHECK-NEXT: ld1 { v1.b }[4], [x13]
+; CHECK-NEXT: add x14, sp, #440
+; CHECK-NEXT: ld1 { v6.b }[4], [x12]
+; CHECK-NEXT: ldr b5, [sp, #216]
+; CHECK-NEXT: add x10, sp, #120
+; CHECK-NEXT: ld1 { v0.b }[5], [x11]
+; CHECK-NEXT: add x11, sp, #56
+; CHECK-NEXT: ldr b7, [sp, #280]
+; CHECK-NEXT: mov v4.b[4], w4
+; CHECK-NEXT: ld1 { v2.b }[4], [x14]
+; CHECK-NEXT: add x14, sp, #224
+; CHECK-NEXT: ld1 { v6.b }[5], [x11]
+; CHECK-NEXT: ld1 { v5.b }[1], [x14]
+; CHECK-NEXT: add x15, sp, #288
+; CHECK-NEXT: ld1 { v1.b }[5], [x10]
+; CHECK-NEXT: add x14, sp, #64
+; CHECK-NEXT: ld1 { v7.b }[1], [x15]
+; CHECK-NEXT: add x9, sp, #192
+; CHECK-NEXT: add x12, sp, #128
+; CHECK-NEXT: ldr b3, [sp, #344]
+; CHECK-NEXT: mov v4.b[5], w5
+; CHECK-NEXT: ld1 { v6.b }[6], [x14]
+; CHECK-NEXT: add x14, sp, #232
+; CHECK-NEXT: ld1 { v0.b }[6], [x9]
+; CHECK-NEXT: ld1 { v5.b }[2], [x14]
+; CHECK-NEXT: ld1 { v1.b }[6], [x12]
+; CHECK-NEXT: add x12, sp, #296
+; CHECK-NEXT: add x13, sp, #352
+; CHECK-NEXT: add x8, sp, #200
+; CHECK-NEXT: ld1 { v7.b }[2], [x12]
+; CHECK-NEXT: ld1 { v3.b }[1], [x13]
+; CHECK-NEXT: add x14, sp, #240
+; CHECK-NEXT: mov v4.b[6], w6
+; CHECK-NEXT: add x10, sp, #448
+; CHECK-NEXT: ld1 { v5.b }[3], [x14]
+; CHECK-NEXT: ld1 { v0.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #304
+; CHECK-NEXT: add x9, sp, #360
+; CHECK-NEXT: ld1 { v2.b }[5], [x10]
+; CHECK-NEXT: ld1 { v7.b }[3], [x8]
+; CHECK-NEXT: add x11, sp, #72
+; CHECK-NEXT: ld1 { v3.b }[2], [x9]
+; CHECK-NEXT: add x8, sp, #248
+; CHECK-NEXT: add x15, sp, #456
+; CHECK-NEXT: mov v4.b[7], w7
+; CHECK-NEXT: ld1 { v6.b }[7], [x11]
+; CHECK-NEXT: ld1 { v5.b }[4], [x8]
+; CHECK-NEXT: add x8, sp, #312
+; CHECK-NEXT: ld1 { v2.b }[6], [x15]
+; CHECK-NEXT: add x9, sp, #368
+; CHECK-NEXT: ld1 { v7.b }[4], [x8]
+; CHECK-NEXT: ld1 { v3.b }[3], [x9]
+; CHECK-NEXT: add x8, sp, #256
+; CHECK-NEXT: ushll v17.8h, v6.8b, #0
+; CHECK-NEXT: add x10, sp, #464
+; CHECK-NEXT: ld1 { v5.b }[5], [x8]
+; CHECK-NEXT: ushll v21.8h, v4.8b, #0
+; CHECK-NEXT: ldr b4, [sp, #208]
+; CHECK-NEXT: add x8, sp, #320
+; CHECK-NEXT: ld1 { v2.b }[7], [x10]
+; CHECK-NEXT: add x9, sp, #376
+; CHECK-NEXT: ld1 { v7.b }[5], [x8]
+; CHECK-NEXT: ushll v6.8h, v4.8b, #0
+; CHECK-NEXT: ldr b4, [sp, #672]
+; CHECK-NEXT: add x10, sp, #680
+; CHECK-NEXT: ld1 { v3.b }[4], [x9]
+; CHECK-NEXT: add x9, sp, #264
+; CHECK-NEXT: ldr b16, [sp, #472]
+; CHECK-NEXT: ld1 { v4.b }[1], [x10]
+; CHECK-NEXT: ld1 { v5.b }[6], [x9]
+; CHECK-NEXT: add x9, sp, #328
+; CHECK-NEXT: ld1 { v7.b }[6], [x9]
+; CHECK-NEXT: add x10, sp, #688
+; CHECK-NEXT: add x9, sp, #272
+; CHECK-NEXT: add x8, sp, #384
+; CHECK-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-NEXT: ldr b19, [sp, #480]
+; CHECK-NEXT: ld1 { v4.b }[2], [x10]
+; CHECK-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-NEXT: add x9, sp, #336
+; CHECK-NEXT: ld1 { v3.b }[5], [x8]
+; CHECK-NEXT: ld1 { v7.b }[7], [x9]
+; CHECK-NEXT: add x9, sp, #696
+; CHECK-NEXT: add x8, sp, #392
+; CHECK-NEXT: smull v20.4s, v6.4h, v16.4h
+; CHECK-NEXT: ldr b16, [sp, #608]
+; CHECK-NEXT: ld1 { v4.b }[3], [x9]
+; CHECK-NEXT: add x9, sp, #488
+; CHECK-NEXT: sshll v22.8h, v5.8b, #0
+; CHECK-NEXT: ld1 { v19.b }[1], [x9]
+; CHECK-NEXT: ld1 { v3.b }[6], [x8]
+; CHECK-NEXT: add x9, sp, #704
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: add x10, sp, #496
+; CHECK-NEXT: add x8, sp, #400
+; CHECK-NEXT: ld1 { v4.b }[4], [x9]
+; CHECK-NEXT: add x9, sp, #616
+; CHECK-NEXT: sshll v18.8h, v7.8b, #0
+; CHECK-NEXT: ld1 { v16.b }[1], [x9]
+; CHECK-NEXT: ld1 { v19.b }[2], [x10]
+; CHECK-NEXT: ld1 { v3.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #712
+; CHECK-NEXT: add x9, sp, #504
+; CHECK-NEXT: smull2 v7.4s, v21.8h, v22.8h
+; CHECK-NEXT: ld1 { v4.b }[5], [x8]
+; CHECK-NEXT: add x8, sp, #624
+; CHECK-NEXT: mov v5.s[0], v20.s[0]
+; CHECK-NEXT: ld1 { v16.b }[2], [x8]
+; CHECK-NEXT: ld1 { v19.b }[3], [x9]
+; CHECK-NEXT: add x8, sp, #720
+; CHECK-NEXT: add x9, sp, #512
+; CHECK-NEXT: ldr b20, [sp, #544]
+; CHECK-NEXT: add x10, sp, #640
+; CHECK-NEXT: ld1 { v4.b }[6], [x8]
+; CHECK-NEXT: add x8, sp, #632
+; CHECK-NEXT: add x11, sp, #520
+; CHECK-NEXT: ld1 { v16.b }[3], [x8]
+; CHECK-NEXT: ld1 { v19.b }[4], [x9]
+; CHECK-NEXT: add x9, sp, #552
+; CHECK-NEXT: smlal v5.4s, v21.4h, v22.4h
+; CHECK-NEXT: ld1 { v20.b }[1], [x9]
+; CHECK-NEXT: ldr b21, [sp, #736]
+; CHECK-NEXT: ldr b22, [sp, #1000]
+; CHECK-NEXT: add x9, sp, #528
+; CHECK-NEXT: ldr b23, [sp, #808]
+; CHECK-NEXT: ld1 { v16.b }[4], [x10]
+; CHECK-NEXT: ld1 { v19.b }[5], [x11]
+; CHECK-NEXT: add x10, sp, #560
+; CHECK-NEXT: ushll v21.8h, v21.8b, #0
+; CHECK-NEXT: sshll v24.8h, v22.8b, #0
+; CHECK-NEXT: ld1 { v20.b }[2], [x10]
+; CHECK-NEXT: smull v6.4s, v17.4h, v18.4h
+; CHECK-NEXT: smull2 v18.4s, v17.8h, v18.8h
+; CHECK-NEXT: movi v17.2d, #0000000000000000
+; CHECK-NEXT: ld1 { v19.b }[6], [x9]
+; CHECK-NEXT: add x9, sp, #568
+; CHECK-NEXT: ldr b22, [sp, #744]
+; CHECK-NEXT: add x11, sp, #816
+; CHECK-NEXT: smull v24.4s, v21.4h, v24.4h
+; CHECK-NEXT: ld1 { v20.b }[3], [x9]
+; CHECK-NEXT: add x10, sp, #752
+; CHECK-NEXT: ld1 { v23.b }[1], [x11]
+; CHECK-NEXT: add x8, sp, #728
+; CHECK-NEXT: ld1 { v22.b }[1], [x10]
+; CHECK-NEXT: add x9, sp, #576
+; CHECK-NEXT: ldr b21, [sp, #936]
+; CHECK-NEXT: add x10, sp, #824
+; CHECK-NEXT: ld1 { v4.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #648
+; CHECK-NEXT: ld1 { v20.b }[4], [x9]
+; CHECK-NEXT: add x9, sp, #760
+; CHECK-NEXT: add x11, sp, #944
+; CHECK-NEXT: ld1 { v23.b }[2], [x10]
+; CHECK-NEXT: mov v17.s[0], v24.s[0]
+; CHECK-NEXT: ldr b24, [sp, #872]
+; CHECK-NEXT: ld1 { v16.b }[5], [x8]
+; CHECK-NEXT: add x8, sp, #536
+; CHECK-NEXT: ld1 { v22.b }[2], [x9]
+; CHECK-NEXT: ld1 { v21.b }[1], [x11]
+; CHECK-NEXT: add x10, sp, #880
+; CHECK-NEXT: ld1 { v19.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #584
+; CHECK-NEXT: add x9, sp, #832
+; CHECK-NEXT: ld1 { v24.b }[1], [x10]
+; CHECK-NEXT: ld1 { v20.b }[5], [x8]
+; CHECK-NEXT: add x8, sp, #768
+; CHECK-NEXT: ld1 { v23.b }[3], [x9]
+; CHECK-NEXT: add x9, sp, #952
+; CHECK-NEXT: ld1 { v22.b }[3], [x8]
+; CHECK-NEXT: add x11, sp, #888
+; CHECK-NEXT: ld1 { v21.b }[2], [x9]
+; CHECK-NEXT: add x8, sp, #592
+; CHECK-NEXT: add x10, sp, #840
+; CHECK-NEXT: ld1 { v24.b }[2], [x11]
+; CHECK-NEXT: add x9, sp, #776
+; CHECK-NEXT: ld1 { v23.b }[4], [x10]
+; CHECK-NEXT: ld1 { v20.b }[6], [x8]
+; CHECK-NEXT: add x8, sp, #960
+; CHECK-NEXT: ld1 { v22.b }[4], [x9]
+; CHECK-NEXT: ld1 { v21.b }[3], [x8]
+; CHECK-NEXT: add x10, sp, #896
+; CHECK-NEXT: add x9, sp, #848
+; CHECK-NEXT: ld1 { v24.b }[3], [x10]
+; CHECK-NEXT: add x8, sp, #784
+; CHECK-NEXT: ld1 { v23.b }[5], [x9]
+; CHECK-NEXT: add x9, sp, #968
+; CHECK-NEXT: ld1 { v22.b }[5], [x8]
+; CHECK-NEXT: add x11, sp, #904
+; CHECK-NEXT: ld1 { v21.b }[4], [x9]
+; CHECK-NEXT: add x8, sp, #600
+; CHECK-NEXT: add x10, sp, #856
+; CHECK-NEXT: ld1 { v24.b }[4], [x11]
+; CHECK-NEXT: add x9, sp, #792
+; CHECK-NEXT: ld1 { v23.b }[6], [x10]
+; CHECK-NEXT: ld1 { v20.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #976
+; CHECK-NEXT: ld1 { v22.b }[6], [x9]
+; CHECK-NEXT: ld1 { v21.b }[5], [x8]
+; CHECK-NEXT: add x10, sp, #912
+; CHECK-NEXT: add x9, sp, #864
+; CHECK-NEXT: ld1 { v24.b }[5], [x10]
+; CHECK-NEXT: add x8, sp, #800
+; CHECK-NEXT: ld1 { v23.b }[7], [x9]
+; CHECK-NEXT: add x9, sp, #984
+; CHECK-NEXT: ld1 { v22.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #656
+; CHECK-NEXT: ld1 { v21.b }[6], [x9]
+; CHECK-NEXT: add x9, sp, #920
+; CHECK-NEXT: ld1 { v16.b }[6], [x8]
+; CHECK-NEXT: ld1 { v24.b }[6], [x9]
+; CHECK-NEXT: add x8, sp, #992
+; CHECK-NEXT: add x13, sp, #136
+; CHECK-NEXT: ushll v19.8h, v19.8b, #0
+; CHECK-NEXT: ushll v20.8h, v20.8b, #0
+; CHECK-NEXT: sshll v22.8h, v22.8b, #0
+; CHECK-NEXT: sshll v23.8h, v23.8b, #0
+; CHECK-NEXT: add x9, sp, #664
+; CHECK-NEXT: ld1 { v21.b }[7], [x8]
+; CHECK-NEXT: add x8, sp, #928
+; CHECK-NEXT: ld1 { v1.b }[7], [x13]
+; CHECK-NEXT: ld1 { v16.b }[7], [x9]
+; CHECK-NEXT: ld1 { v24.b }[7], [x8]
+; CHECK-NEXT: smlal v17.4s, v19.4h, v22.4h
+; CHECK-NEXT: smull2 v19.4s, v19.8h, v22.8h
+; CHECK-NEXT: smull v22.4s, v20.4h, v23.4h
+; CHECK-NEXT: smull2 v20.4s, v20.8h, v23.8h
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-NEXT: sshll v2.8h, v2.8b, #0
+; CHECK-NEXT: ushll v4.8h, v4.8b, #0
+; CHECK-NEXT: sshll v21.8h, v21.8b, #0
+; CHECK-NEXT: ushll v16.8h, v16.8b, #0
+; CHECK-NEXT: sshll v23.8h, v24.8b, #0
+; CHECK-NEXT: smlal2 v7.4s, v1.8h, v3.8h
+; CHECK-NEXT: smlal v5.4s, v1.4h, v3.4h
+; CHECK-NEXT: smlal2 v18.4s, v0.8h, v2.8h
+; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h
+; CHECK-NEXT: smlal2 v20.4s, v4.8h, v21.8h
+; CHECK-NEXT: smlal v22.4s, v4.4h, v21.4h
+; CHECK-NEXT: smlal2 v19.4s, v16.8h, v23.8h
+; CHECK-NEXT: smlal v17.4s, v16.4h, v23.4h
+; CHECK-NEXT: add v0.4s, v7.4s, v18.4s
+; CHECK-NEXT: add v1.4s, v5.4s, v6.4s
+; CHECK-NEXT: add v2.4s, v19.4s, v20.4s
+; CHECK-NEXT: add v3.4s, v17.4s, v22.4s
+; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: add v1.4s, v3.4s, v2.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+entry:
+ %az = zext <33 x i8> %a to <33 x i32>
+ %bz = sext <33 x i8> %b to <33 x i32>
+ %m1 = mul nuw nsw <33 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1)
+ %cz = zext <33 x i8> %c to <33 x i32>
+ %dz = sext <33 x i8> %d to <33 x i32>
+ %m2 = mul nuw nsw <33 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v48i8:
; CHECK-SD: // %bb.0: // %entry
@@ -3243,1003 +5230,2082 @@ define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48
; CHECK-SD-NEXT: add x8, sp, #1120
; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9]
; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #736
-; CHECK-SD-NEXT: ldr b7, [sp, #1232]
-; CHECK-SD-NEXT: ldr b16, [sp, #848]
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8]
-; CHECK-SD-NEXT: add x9, sp, #1240
-; CHECK-SD-NEXT: add x10, sp, #856
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1128
-; CHECK-SD-NEXT: add x11, sp, #744
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1248
-; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #864
-; CHECK-SD-NEXT: add x9, sp, #144
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1136
-; CHECK-SD-NEXT: add x12, sp, #752
-; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1256
-; CHECK-SD-NEXT: add x10, sp, #872
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1144
-; CHECK-SD-NEXT: add x11, sp, #760
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1264
-; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #880
-; CHECK-SD-NEXT: add x9, sp, #152
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1152
-; CHECK-SD-NEXT: add x12, sp, #768
-; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1272
-; CHECK-SD-NEXT: add x10, sp, #888
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1160
-; CHECK-SD-NEXT: add x11, sp, #776
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1280
-; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11]
-; CHECK-SD-NEXT: add x11, sp, #896
-; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #736
+; CHECK-SD-NEXT: ldr b7, [sp, #1232]
+; CHECK-SD-NEXT: ldr b16, [sp, #848]
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8]
+; CHECK-SD-NEXT: add x9, sp, #1240
+; CHECK-SD-NEXT: add x10, sp, #856
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1128
+; CHECK-SD-NEXT: add x11, sp, #744
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1248
+; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #864
+; CHECK-SD-NEXT: add x9, sp, #144
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1136
+; CHECK-SD-NEXT: add x12, sp, #752
+; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1256
+; CHECK-SD-NEXT: add x10, sp, #872
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1144
+; CHECK-SD-NEXT: add x11, sp, #760
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1264
+; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #880
+; CHECK-SD-NEXT: add x9, sp, #152
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1152
+; CHECK-SD-NEXT: add x12, sp, #768
+; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1272
+; CHECK-SD-NEXT: add x10, sp, #888
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1160
+; CHECK-SD-NEXT: add x11, sp, #776
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1280
+; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11]
+; CHECK-SD-NEXT: add x11, sp, #896
+; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1168
+; CHECK-SD-NEXT: add x12, sp, #784
+; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1288
+; CHECK-SD-NEXT: add x10, sp, #904
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1176
+; CHECK-SD-NEXT: add x11, sp, #792
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1296
+; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11]
+; CHECK-SD-NEXT: add x11, sp, #912
+; CHECK-SD-NEXT: add x9, sp, #168
+; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1184
+; CHECK-SD-NEXT: add x12, sp, #800
+; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1304
+; CHECK-SD-NEXT: add x10, sp, #920
+; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1192
+; CHECK-SD-NEXT: add x11, sp, #808
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1312
+; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11]
+; CHECK-SD-NEXT: add x11, sp, #928
+; CHECK-SD-NEXT: add x9, sp, #176
+; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1200
+; CHECK-SD-NEXT: add x12, sp, #816
+; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1320
+; CHECK-SD-NEXT: add x10, sp, #936
+; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1208
+; CHECK-SD-NEXT: add x11, sp, #824
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1328
+; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11]
+; CHECK-SD-NEXT: add x11, sp, #944
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1216
+; CHECK-SD-NEXT: add x12, sp, #832
+; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1336
+; CHECK-SD-NEXT: add x10, sp, #952
+; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1224
+; CHECK-SD-NEXT: add x11, sp, #840
+; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #192
+; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11]
+; CHECK-SD-NEXT: add x10, sp, #1344
+; CHECK-SD-NEXT: add x11, sp, #960
+; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11]
+; CHECK-SD-NEXT: add x9, sp, #584
+; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b
+; CHECK-SD-NEXT: add x8, sp, #200
+; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b
+; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1352
+; CHECK-SD-NEXT: add x10, sp, #968
+; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10]
+; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_sdot_v48i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: .cfi_offset w29, -16
+; CHECK-GI-NEXT: ldr w11, [sp, #80]
+; CHECK-GI-NEXT: ldr w10, [sp, #208]
+; CHECK-GI-NEXT: fmov s0, w0
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: ldr w12, [sp, #344]
+; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: fmov s1, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #336]
+; CHECK-GI-NEXT: fmov s2, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #464]
+; CHECK-GI-NEXT: ldr w9, [sp, #216]
+; CHECK-GI-NEXT: mov v0.b[1], w1
+; CHECK-GI-NEXT: fmov s3, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #600]
+; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #592]
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: mov v2.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #472]
+; CHECK-GI-NEXT: ldr w10, [sp, #608]
+; CHECK-GI-NEXT: mov v3.b[1], w12
+; CHECK-GI-NEXT: fmov s5, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #96]
+; CHECK-GI-NEXT: mov v4.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #224]
+; CHECK-GI-NEXT: mov v0.b[2], w2
+; CHECK-GI-NEXT: mov v1.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #352]
+; CHECK-GI-NEXT: ldr w12, [sp, #848]
+; CHECK-GI-NEXT: mov v2.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: mov v5.b[1], w11
+; CHECK-GI-NEXT: mov v3.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: ldr w11, [sp, #16]
+; CHECK-GI-NEXT: mov v4.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #232]
+; CHECK-GI-NEXT: mov v0.b[3], w3
+; CHECK-GI-NEXT: mov v1.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #360]
+; CHECK-GI-NEXT: fmov s7, w12
+; CHECK-GI-NEXT: mov v2.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #488]
+; CHECK-GI-NEXT: mov v5.b[2], w10
+; CHECK-GI-NEXT: mov v3.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #112]
+; CHECK-GI-NEXT: ldr w10, [sp, #616]
+; CHECK-GI-NEXT: mov v4.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #240]
+; CHECK-GI-NEXT: mov v0.b[4], w4
+; CHECK-GI-NEXT: mov v1.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #368]
+; CHECK-GI-NEXT: ldr w12, [sp, #1112]
+; CHECK-GI-NEXT: mov v2.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #496]
+; CHECK-GI-NEXT: mov v5.b[3], w10
+; CHECK-GI-NEXT: mov v3.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #120]
+; CHECK-GI-NEXT: ldr w10, [sp, #624]
+; CHECK-GI-NEXT: mov v4.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #248]
+; CHECK-GI-NEXT: mov v0.b[5], w5
+; CHECK-GI-NEXT: mov v1.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #376]
+; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #504]
+; CHECK-GI-NEXT: mov v5.b[4], w10
+; CHECK-GI-NEXT: mov v3.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #128]
+; CHECK-GI-NEXT: ldr w10, [sp, #632]
+; CHECK-GI-NEXT: mov v4.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: mov v0.b[6], w6
+; CHECK-GI-NEXT: mov v1.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #384]
+; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #512]
+; CHECK-GI-NEXT: mov v5.b[5], w10
+; CHECK-GI-NEXT: mov v3.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #136]
+; CHECK-GI-NEXT: ldr w10, [sp, #640]
+; CHECK-GI-NEXT: mov v4.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #264]
+; CHECK-GI-NEXT: mov v0.b[7], w7
+; CHECK-GI-NEXT: mov v1.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #392]
+; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #520]
+; CHECK-GI-NEXT: mov v5.b[6], w10
+; CHECK-GI-NEXT: mov v3.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #144]
+; CHECK-GI-NEXT: ldr w10, [sp, #648]
+; CHECK-GI-NEXT: mov v4.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #272]
+; CHECK-GI-NEXT: mov v0.b[8], w11
+; CHECK-GI-NEXT: mov v1.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #400]
+; CHECK-GI-NEXT: ldr w11, [sp, #24]
+; CHECK-GI-NEXT: mov v2.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #528]
+; CHECK-GI-NEXT: mov v5.b[7], w10
+; CHECK-GI-NEXT: mov v3.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #152]
+; CHECK-GI-NEXT: ldr w10, [sp, #656]
+; CHECK-GI-NEXT: mov v4.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #280]
+; CHECK-GI-NEXT: mov v0.b[9], w11
+; CHECK-GI-NEXT: mov v1.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #408]
+; CHECK-GI-NEXT: ldr w11, [sp, #32]
+; CHECK-GI-NEXT: mov v2.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #536]
+; CHECK-GI-NEXT: mov v5.b[8], w10
+; CHECK-GI-NEXT: mov v3.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #160]
+; CHECK-GI-NEXT: ldr w10, [sp, #664]
+; CHECK-GI-NEXT: mov v4.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #288]
+; CHECK-GI-NEXT: mov v0.b[10], w11
+; CHECK-GI-NEXT: mov v1.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #416]
+; CHECK-GI-NEXT: ldr w11, [sp, #40]
+; CHECK-GI-NEXT: mov v2.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #544]
+; CHECK-GI-NEXT: mov v5.b[9], w10
+; CHECK-GI-NEXT: mov v3.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #168]
+; CHECK-GI-NEXT: ldr w10, [sp, #672]
+; CHECK-GI-NEXT: mov v4.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #296]
+; CHECK-GI-NEXT: mov v0.b[11], w11
+; CHECK-GI-NEXT: mov v1.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #424]
+; CHECK-GI-NEXT: ldr w11, [sp, #48]
+; CHECK-GI-NEXT: mov v2.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #552]
+; CHECK-GI-NEXT: mov v5.b[10], w10
+; CHECK-GI-NEXT: mov v3.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #176]
+; CHECK-GI-NEXT: ldr w10, [sp, #680]
+; CHECK-GI-NEXT: mov v4.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #304]
+; CHECK-GI-NEXT: mov v0.b[12], w11
+; CHECK-GI-NEXT: mov v1.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #432]
+; CHECK-GI-NEXT: ldr w11, [sp, #56]
+; CHECK-GI-NEXT: mov v2.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #560]
+; CHECK-GI-NEXT: mov v5.b[11], w10
+; CHECK-GI-NEXT: mov v3.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #184]
+; CHECK-GI-NEXT: ldr w10, [sp, #688]
+; CHECK-GI-NEXT: mov v4.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #312]
+; CHECK-GI-NEXT: mov v0.b[13], w11
+; CHECK-GI-NEXT: mov v1.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #440]
+; CHECK-GI-NEXT: ldr w11, [sp, #64]
+; CHECK-GI-NEXT: mov v2.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #568]
+; CHECK-GI-NEXT: mov v5.b[12], w10
+; CHECK-GI-NEXT: mov v3.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #192]
+; CHECK-GI-NEXT: ldr w10, [sp, #696]
+; CHECK-GI-NEXT: mov v4.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #320]
+; CHECK-GI-NEXT: mov v0.b[14], w11
+; CHECK-GI-NEXT: mov v1.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #448]
+; CHECK-GI-NEXT: ldr w11, [sp, #72]
+; CHECK-GI-NEXT: mov v2.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #576]
+; CHECK-GI-NEXT: mov v5.b[13], w10
+; CHECK-GI-NEXT: mov v3.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #720]
+; CHECK-GI-NEXT: ldr w10, [sp, #704]
+; CHECK-GI-NEXT: mov v4.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #728]
+; CHECK-GI-NEXT: mov v0.b[15], w11
+; CHECK-GI-NEXT: fmov s6, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #328]
+; CHECK-GI-NEXT: ldr w11, [sp, #456]
+; CHECK-GI-NEXT: mov v5.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #200]
+; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[15], w8
+; CHECK-GI-NEXT: mov v3.b[15], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #736]
+; CHECK-GI-NEXT: mov v6.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #584]
+; CHECK-GI-NEXT: ldr w8, [sp, #856]
+; CHECK-GI-NEXT: mov v1.b[15], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #712]
+; CHECK-GI-NEXT: mov v4.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #976]
+; CHECK-GI-NEXT: mov v7.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1232]
+; CHECK-GI-NEXT: mov v5.b[15], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #984]
+; CHECK-GI-NEXT: mov v6.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1104]
+; CHECK-GI-NEXT: fmov s16, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1360]
+; CHECK-GI-NEXT: fmov s18, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1368]
+; CHECK-GI-NEXT: fmov s17, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1240]
+; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b
+; CHECK-GI-NEXT: mov v16.b[1], w10
+; CHECK-GI-NEXT: fmov s19, w9
+; CHECK-GI-NEXT: ldr w10, [sp, #864]
+; CHECK-GI-NEXT: mov v18.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: ldr w9, [sp, #1120]
+; CHECK-GI-NEXT: mov v17.b[1], w12
+; CHECK-GI-NEXT: mov v7.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1248]
+; CHECK-GI-NEXT: mov v19.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT: mov v16.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #872]
+; CHECK-GI-NEXT: addv s0, v20.4s
+; CHECK-GI-NEXT: mov v6.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1000]
+; CHECK-GI-NEXT: mov v18.b[2], w10
+; CHECK-GI-NEXT: mov v17.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1376]
+; CHECK-GI-NEXT: ldr w10, [sp, #1128]
+; CHECK-GI-NEXT: mov v7.b[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #880]
+; CHECK-GI-NEXT: addv s1, v21.4s
+; CHECK-GI-NEXT: mov v19.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #752]
+; CHECK-GI-NEXT: mov v16.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1256]
+; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT: mov v17.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1384]
+; CHECK-GI-NEXT: mov v6.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1008]
+; CHECK-GI-NEXT: mov v18.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1136]
+; CHECK-GI-NEXT: mov v19.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #760]
+; CHECK-GI-NEXT: mov v7.b[4], w11
+; CHECK-GI-NEXT: mov v16.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1264]
+; CHECK-GI-NEXT: ldr w11, [sp, #888]
+; CHECK-GI-NEXT: mov v17.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1392]
+; CHECK-GI-NEXT: mov v6.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1016]
+; CHECK-GI-NEXT: mov v18.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1144]
+; CHECK-GI-NEXT: mov v19.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #768]
+; CHECK-GI-NEXT: mov v7.b[5], w11
+; CHECK-GI-NEXT: mov v16.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1272]
+; CHECK-GI-NEXT: ldr w11, [sp, #896]
+; CHECK-GI-NEXT: mov v17.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1400]
+; CHECK-GI-NEXT: mov v6.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1024]
+; CHECK-GI-NEXT: mov v18.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1152]
+; CHECK-GI-NEXT: mov v19.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #776]
+; CHECK-GI-NEXT: mov v7.b[6], w11
+; CHECK-GI-NEXT: mov v16.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1280]
+; CHECK-GI-NEXT: ldr w11, [sp, #904]
+; CHECK-GI-NEXT: mov v17.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1408]
+; CHECK-GI-NEXT: mov v6.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1032]
+; CHECK-GI-NEXT: mov v18.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1160]
+; CHECK-GI-NEXT: mov v19.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #784]
+; CHECK-GI-NEXT: mov v7.b[7], w11
+; CHECK-GI-NEXT: mov v16.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1288]
+; CHECK-GI-NEXT: ldr w11, [sp, #912]
+; CHECK-GI-NEXT: mov v17.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1416]
+; CHECK-GI-NEXT: mov v6.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1040]
+; CHECK-GI-NEXT: mov v18.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1168]
+; CHECK-GI-NEXT: mov v19.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #792]
+; CHECK-GI-NEXT: mov v7.b[8], w11
+; CHECK-GI-NEXT: mov v16.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1296]
+; CHECK-GI-NEXT: ldr w11, [sp, #920]
+; CHECK-GI-NEXT: mov v17.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1424]
+; CHECK-GI-NEXT: mov v6.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1048]
+; CHECK-GI-NEXT: mov v18.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1176]
+; CHECK-GI-NEXT: mov v19.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #800]
+; CHECK-GI-NEXT: mov v7.b[9], w11
+; CHECK-GI-NEXT: mov v16.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1304]
+; CHECK-GI-NEXT: ldr w11, [sp, #928]
+; CHECK-GI-NEXT: mov v17.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1432]
+; CHECK-GI-NEXT: mov v6.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1056]
+; CHECK-GI-NEXT: mov v18.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1184]
+; CHECK-GI-NEXT: mov v19.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #808]
+; CHECK-GI-NEXT: mov v7.b[10], w11
+; CHECK-GI-NEXT: mov v16.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1312]
+; CHECK-GI-NEXT: ldr w11, [sp, #936]
+; CHECK-GI-NEXT: mov v17.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1440]
+; CHECK-GI-NEXT: mov v6.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1064]
+; CHECK-GI-NEXT: mov v18.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1192]
+; CHECK-GI-NEXT: mov v19.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #816]
+; CHECK-GI-NEXT: mov v7.b[11], w11
+; CHECK-GI-NEXT: mov v16.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1320]
+; CHECK-GI-NEXT: ldr w11, [sp, #944]
+; CHECK-GI-NEXT: mov v17.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1448]
+; CHECK-GI-NEXT: mov v6.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1072]
+; CHECK-GI-NEXT: mov v18.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1200]
+; CHECK-GI-NEXT: mov v19.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #824]
+; CHECK-GI-NEXT: mov v7.b[12], w11
+; CHECK-GI-NEXT: mov v16.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1328]
+; CHECK-GI-NEXT: ldr w11, [sp, #952]
+; CHECK-GI-NEXT: mov v17.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1456]
+; CHECK-GI-NEXT: mov v6.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1080]
+; CHECK-GI-NEXT: mov v18.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1208]
+; CHECK-GI-NEXT: mov v19.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #832]
+; CHECK-GI-NEXT: mov v7.b[13], w11
+; CHECK-GI-NEXT: mov v16.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1336]
+; CHECK-GI-NEXT: ldr w11, [sp, #960]
+; CHECK-GI-NEXT: mov v17.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1464]
+; CHECK-GI-NEXT: mov v6.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1088]
+; CHECK-GI-NEXT: mov v18.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1216]
+; CHECK-GI-NEXT: mov v19.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #840]
+; CHECK-GI-NEXT: mov v7.b[14], w11
+; CHECK-GI-NEXT: mov v16.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1344]
+; CHECK-GI-NEXT: ldr w11, [sp, #968]
+; CHECK-GI-NEXT: mov v17.b[14], w9
+; CHECK-GI-NEXT: mov v6.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1096]
+; CHECK-GI-NEXT: mov v18.b[14], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #1472]
+; CHECK-GI-NEXT: ldr w10, [sp, #1224]
+; CHECK-GI-NEXT: mov v7.b[15], w11
+; CHECK-GI-NEXT: addv s4, v25.4s
+; CHECK-GI-NEXT: mov v16.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1352]
+; CHECK-GI-NEXT: mov v19.b[14], w9
+; CHECK-GI-NEXT: mov v17.b[15], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #1480]
+; CHECK-GI-NEXT: mov v18.b[15], w8
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w11, s4
+; CHECK-GI-NEXT: mov v19.b[15], w9
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b
+; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT: add w8, w8, w11
+; CHECK-GI-NEXT: addv s2, v22.4s
+; CHECK-GI-NEXT: addv s3, v23.4s
+; CHECK-GI-NEXT: addv s5, v24.4s
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: fmov w10, s3
+; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: fmov w10, s5
+; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: ret
+entry:
+ %az = sext <48 x i8> %a to <48 x i32>
+ %bz = sext <48 x i8> %b to <48 x i32>
+ %m1 = mul nuw nsw <48 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
+ %cz = sext <48 x i8> %c to <48 x i32>
+ %dz = sext <48 x i8> %d to <48 x i32>
+ %m2 = mul nuw nsw <48 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT: .cfi_offset w29, -16
+; CHECK-SD-NEXT: ldr b5, [sp, #208]
+; CHECK-SD-NEXT: add x8, sp, #216
+; CHECK-SD-NEXT: fmov s0, w0
+; CHECK-SD-NEXT: ldr b4, [sp, #976]
+; CHECK-SD-NEXT: add x9, sp, #984
+; CHECK-SD-NEXT: add x12, sp, #328
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #224
+; CHECK-SD-NEXT: movi v1.16b, #1
+; CHECK-SD-NEXT: mov v0.b[1], w1
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9]
+; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT: add x11, sp, #992
+; CHECK-SD-NEXT: ldr b6, [sp, #720]
+; CHECK-SD-NEXT: ldr b7, [sp, #80]
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #232
+; CHECK-SD-NEXT: add x13, sp, #88
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT: add x13, sp, #856
+; CHECK-SD-NEXT: mov v0.b[2], w2
+; CHECK-SD-NEXT: add x14, sp, #1008
+; CHECK-SD-NEXT: add x15, sp, #872
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #240
+; CHECK-SD-NEXT: add x16, sp, #888
+; CHECK-SD-NEXT: add x10, sp, #16
+; CHECK-SD-NEXT: add x9, sp, #24
+; CHECK-SD-NEXT: add x11, sp, #40
+; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #248
+; CHECK-SD-NEXT: mov v0.b[3], w3
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT: add x8, sp, #256
+; CHECK-SD-NEXT: mov v0.b[4], w4
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #264
+; CHECK-SD-NEXT: mov v0.b[5], w5
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
+; CHECK-SD-NEXT: add x8, sp, #272
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8]
+; CHECK-SD-NEXT: add x8, sp, #280
+; CHECK-SD-NEXT: mov v0.b[6], w6
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8]
+; CHECK-SD-NEXT: add x8, sp, #288
+; CHECK-SD-NEXT: mov v0.b[7], w7
+; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #296
+; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10]
+; CHECK-SD-NEXT: add x10, sp, #128
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #304
+; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #312
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #320
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT: add x8, sp, #32
+; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #144
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12]
+; CHECK-SD-NEXT: add x12, sp, #728
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1000
+; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11]
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12]
+; CHECK-SD-NEXT: add x12, sp, #736
+; CHECK-SD-NEXT: add x11, sp, #920
+; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT: ldr b5, [sp, #848]
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT: add x12, sp, #48
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13]
+; CHECK-SD-NEXT: add x13, sp, #744
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14]
+; CHECK-SD-NEXT: add x14, sp, #96
+; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13]
+; CHECK-SD-NEXT: add x13, sp, #864
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14]
+; CHECK-SD-NEXT: add x14, sp, #1016
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
+; CHECK-SD-NEXT: add x13, sp, #752
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14]
+; CHECK-SD-NEXT: add x14, sp, #104
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
+; CHECK-SD-NEXT: add x13, sp, #1024
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14]
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15]
+; CHECK-SD-NEXT: add x15, sp, #760
+; CHECK-SD-NEXT: add x14, sp, #112
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
+; CHECK-SD-NEXT: add x13, sp, #880
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15]
+; CHECK-SD-NEXT: add x15, sp, #1032
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT: add x14, sp, #768
+; CHECK-SD-NEXT: add x13, sp, #120
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15]
+; CHECK-SD-NEXT: add x15, sp, #1040
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14]
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT: add x13, sp, #776
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16]
+; CHECK-SD-NEXT: add x14, sp, #1048
+; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15]
+; CHECK-SD-NEXT: add x15, sp, #896
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13]
; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1168
-; CHECK-SD-NEXT: add x12, sp, #784
-; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1288
-; CHECK-SD-NEXT: add x10, sp, #904
+; CHECK-SD-NEXT: add x10, sp, #784
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15]
+; CHECK-SD-NEXT: add x13, sp, #1056
+; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
+; CHECK-SD-NEXT: add x14, sp, #904
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10]
; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1176
-; CHECK-SD-NEXT: add x11, sp, #792
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1296
-; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11]
-; CHECK-SD-NEXT: add x11, sp, #912
-; CHECK-SD-NEXT: add x9, sp, #168
-; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1184
-; CHECK-SD-NEXT: add x12, sp, #800
-; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1304
-; CHECK-SD-NEXT: add x10, sp, #920
-; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1192
-; CHECK-SD-NEXT: add x11, sp, #808
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1312
-; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11]
-; CHECK-SD-NEXT: add x11, sp, #928
-; CHECK-SD-NEXT: add x9, sp, #176
+; CHECK-SD-NEXT: add x9, sp, #792
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14]
+; CHECK-SD-NEXT: add x10, sp, #1064
+; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13]
+; CHECK-SD-NEXT: add x13, sp, #912
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
+; CHECK-SD-NEXT: add x9, sp, #800
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13]
+; CHECK-SD-NEXT: add x8, sp, #152
+; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1072
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
+; CHECK-SD-NEXT: add x9, sp, #808
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
+; CHECK-SD-NEXT: add x8, sp, #56
+; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10]
+; CHECK-SD-NEXT: add x10, sp, #160
+; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9]
+; CHECK-SD-NEXT: add x9, sp, #928
; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1200
-; CHECK-SD-NEXT: add x12, sp, #816
-; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9]
+; CHECK-SD-NEXT: add x10, sp, #1080
+; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
+; CHECK-SD-NEXT: add x8, sp, #816
+; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
+; CHECK-SD-NEXT: add x9, sp, #168
+; CHECK-SD-NEXT: add x10, sp, #176
; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1320
-; CHECK-SD-NEXT: add x10, sp, #936
+; CHECK-SD-NEXT: add x8, sp, #936
; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1208
-; CHECK-SD-NEXT: add x11, sp, #824
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1328
-; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11]
-; CHECK-SD-NEXT: add x11, sp, #944
-; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: add x9, sp, #1088
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #64
+; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
+; CHECK-SD-NEXT: add x9, sp, #824
+; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
+; CHECK-SD-NEXT: add x9, sp, #944
; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1216
-; CHECK-SD-NEXT: add x12, sp, #832
-; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9]
+; CHECK-SD-NEXT: add x10, sp, #1096
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
+; CHECK-SD-NEXT: add x8, sp, #832
+; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10]
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: add x10, sp, #72
; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1336
-; CHECK-SD-NEXT: add x10, sp, #952
+; CHECK-SD-NEXT: add x8, sp, #952
; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1224
-; CHECK-SD-NEXT: add x11, sp, #840
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #840
+; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
+; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b
+; CHECK-SD-NEXT: add x9, sp, #192
; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #192
-; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11]
-; CHECK-SD-NEXT: add x10, sp, #1344
-; CHECK-SD-NEXT: add x11, sp, #960
-; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11]
-; CHECK-SD-NEXT: add x9, sp, #584
-; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b
-; CHECK-SD-NEXT: add x8, sp, #200
-; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b
-; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1352
-; CHECK-SD-NEXT: add x10, sp, #968
-; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10]
-; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b
-; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
-; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: add x8, sp, #960
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT: add x8, sp, #200
+; CHECK-SD-NEXT: add x9, sp, #968
+; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
+; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b
+; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT: ret
;
-; CHECK-GI-LABEL: test_sdot_v48i8_double:
+; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w11, [sp, #80]
-; CHECK-GI-NEXT: ldr w10, [sp, #208]
+; CHECK-GI-NEXT: ldr w10, [sp, #80]
+; CHECK-GI-NEXT: ldr w11, [sp, #208]
; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: ldr w12, [sp, #344]
-; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT: fmov s1, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #336]
-; CHECK-GI-NEXT: fmov s2, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #464]
-; CHECK-GI-NEXT: ldr w9, [sp, #216]
+; CHECK-GI-NEXT: ldr w9, [sp, #88]
+; CHECK-GI-NEXT: ldr w12, [sp, #728]
+; CHECK-GI-NEXT: movi v6.16b, #1
+; CHECK-GI-NEXT: fmov s1, w10
+; CHECK-GI-NEXT: fmov s2, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #720]
+; CHECK-GI-NEXT: ldr w10, [sp, #216]
; CHECK-GI-NEXT: mov v0.b[1], w1
+; CHECK-GI-NEXT: ldr w13, [sp, #856]
; CHECK-GI-NEXT: fmov s3, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #600]
-; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #592]
-; CHECK-GI-NEXT: fmov s4, w10
-; CHECK-GI-NEXT: mov v2.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #472]
-; CHECK-GI-NEXT: ldr w10, [sp, #608]
-; CHECK-GI-NEXT: mov v3.b[1], w12
-; CHECK-GI-NEXT: fmov s5, w8
; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: mov v4.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #224]
+; CHECK-GI-NEXT: ldr w11, [sp, #224]
+; CHECK-GI-NEXT: mov v1.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #848]
+; CHECK-GI-NEXT: mov v2.b[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #976]
+; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
+; CHECK-GI-NEXT: fmov s4, w9
+; CHECK-GI-NEXT: mov v3.b[1], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #984]
+; CHECK-GI-NEXT: fmov s5, w10
; CHECK-GI-NEXT: mov v0.b[2], w2
+; CHECK-GI-NEXT: ldr w10, [sp, #736]
; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #352]
-; CHECK-GI-NEXT: ldr w12, [sp, #848]
-; CHECK-GI-NEXT: mov v2.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #480]
-; CHECK-GI-NEXT: mov v5.b[1], w11
-; CHECK-GI-NEXT: mov v3.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #104]
-; CHECK-GI-NEXT: ldr w11, [sp, #16]
-; CHECK-GI-NEXT: mov v4.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #232]
+; CHECK-GI-NEXT: ldr w8, [sp, #864]
+; CHECK-GI-NEXT: mov v2.b[2], w11
+; CHECK-GI-NEXT: mov v4.b[1], w13
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: ldr w12, [sp, #776]
+; CHECK-GI-NEXT: mov v5.b[1], w9
+; CHECK-GI-NEXT: mov v3.b[2], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #104]
+; CHECK-GI-NEXT: ldr w10, [sp, #232]
; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: mov v1.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #360]
-; CHECK-GI-NEXT: fmov s7, w12
-; CHECK-GI-NEXT: mov v2.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #488]
-; CHECK-GI-NEXT: mov v5.b[2], w10
+; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #872]
+; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v4.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: mov v2.b[3], w10
+; CHECK-GI-NEXT: mov v5.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1000]
+; CHECK-GI-NEXT: ldr w10, [sp, #240]
; CHECK-GI-NEXT: mov v3.b[3], w8
; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: ldr w10, [sp, #616]
-; CHECK-GI-NEXT: mov v4.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #240]
; CHECK-GI-NEXT: mov v0.b[4], w4
+; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v4.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #752]
; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #368]
-; CHECK-GI-NEXT: ldr w12, [sp, #1112]
-; CHECK-GI-NEXT: mov v2.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #496]
-; CHECK-GI-NEXT: mov v5.b[3], w10
-; CHECK-GI-NEXT: mov v3.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #120]
-; CHECK-GI-NEXT: ldr w10, [sp, #624]
-; CHECK-GI-NEXT: mov v4.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #248]
+; CHECK-GI-NEXT: ldr w8, [sp, #880]
+; CHECK-GI-NEXT: mov v5.b[3], w11
+; CHECK-GI-NEXT: mov v2.b[4], w10
+; CHECK-GI-NEXT: mov v3.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #120]
+; CHECK-GI-NEXT: ldr w11, [sp, #1008]
+; CHECK-GI-NEXT: ldr w10, [sp, #248]
; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v1.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #376]
-; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #504]
-; CHECK-GI-NEXT: mov v5.b[4], w10
+; CHECK-GI-NEXT: mov v4.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #760]
+; CHECK-GI-NEXT: mov v1.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #888]
+; CHECK-GI-NEXT: mov v5.b[4], w11
+; CHECK-GI-NEXT: mov v2.b[5], w10
; CHECK-GI-NEXT: mov v3.b[5], w8
; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: ldr w10, [sp, #632]
-; CHECK-GI-NEXT: mov v4.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: ldr w11, [sp, #1016]
+; CHECK-GI-NEXT: ldr w10, [sp, #256]
; CHECK-GI-NEXT: mov v0.b[6], w6
+; CHECK-GI-NEXT: mov v4.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #768]
; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #384]
-; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #512]
-; CHECK-GI-NEXT: mov v5.b[5], w10
-; CHECK-GI-NEXT: mov v3.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #136]
-; CHECK-GI-NEXT: ldr w10, [sp, #640]
-; CHECK-GI-NEXT: mov v4.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #264]
+; CHECK-GI-NEXT: ldr w8, [sp, #896]
+; CHECK-GI-NEXT: mov v5.b[5], w11
+; CHECK-GI-NEXT: mov v2.b[6], w10
+; CHECK-GI-NEXT: mov v3.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #136]
+; CHECK-GI-NEXT: ldr w11, [sp, #1024]
+; CHECK-GI-NEXT: ldr w10, [sp, #264]
; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v1.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #392]
-; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #520]
-; CHECK-GI-NEXT: mov v5.b[6], w10
-; CHECK-GI-NEXT: mov v3.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #144]
-; CHECK-GI-NEXT: ldr w10, [sp, #648]
+; CHECK-GI-NEXT: mov v4.b[6], w8
+; CHECK-GI-NEXT: mov v1.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #904]
+; CHECK-GI-NEXT: mov v5.b[6], w11
+; CHECK-GI-NEXT: mov v2.b[7], w10
+; CHECK-GI-NEXT: ldr w8, [sp, #16]
+; CHECK-GI-NEXT: mov v3.b[7], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #144]
+; CHECK-GI-NEXT: ldr w12, [sp, #1032]
+; CHECK-GI-NEXT: mov v0.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #784]
+; CHECK-GI-NEXT: ldr w11, [sp, #272]
; CHECK-GI-NEXT: mov v4.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #272]
-; CHECK-GI-NEXT: mov v0.b[8], w11
-; CHECK-GI-NEXT: mov v1.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #400]
-; CHECK-GI-NEXT: ldr w11, [sp, #24]
-; CHECK-GI-NEXT: mov v2.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #528]
-; CHECK-GI-NEXT: mov v5.b[7], w10
+; CHECK-GI-NEXT: mov v1.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #912]
+; CHECK-GI-NEXT: mov v5.b[7], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #24]
+; CHECK-GI-NEXT: ldr w12, [sp, #1040]
; CHECK-GI-NEXT: mov v3.b[8], w8
; CHECK-GI-NEXT: ldr w8, [sp, #152]
-; CHECK-GI-NEXT: ldr w10, [sp, #656]
-; CHECK-GI-NEXT: mov v4.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #280]
-; CHECK-GI-NEXT: mov v0.b[9], w11
+; CHECK-GI-NEXT: mov v2.b[8], w11
+; CHECK-GI-NEXT: mov v0.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #792]
+; CHECK-GI-NEXT: ldr w11, [sp, #280]
+; CHECK-GI-NEXT: mov v4.b[8], w10
; CHECK-GI-NEXT: mov v1.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #408]
-; CHECK-GI-NEXT: ldr w11, [sp, #32]
-; CHECK-GI-NEXT: mov v2.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #536]
-; CHECK-GI-NEXT: mov v5.b[8], w10
-; CHECK-GI-NEXT: mov v3.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: ldr w10, [sp, #664]
-; CHECK-GI-NEXT: mov v4.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #288]
-; CHECK-GI-NEXT: mov v0.b[10], w11
-; CHECK-GI-NEXT: mov v1.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #416]
-; CHECK-GI-NEXT: ldr w11, [sp, #40]
-; CHECK-GI-NEXT: mov v2.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #544]
-; CHECK-GI-NEXT: mov v5.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #920]
+; CHECK-GI-NEXT: mov v5.b[8], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #32]
+; CHECK-GI-NEXT: ldr w12, [sp, #1048]
+; CHECK-GI-NEXT: mov v3.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #160]
+; CHECK-GI-NEXT: mov v2.b[9], w11
+; CHECK-GI-NEXT: mov v0.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #800]
+; CHECK-GI-NEXT: ldr w11, [sp, #288]
+; CHECK-GI-NEXT: mov v4.b[9], w10
+; CHECK-GI-NEXT: mov v1.b[10], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #928]
+; CHECK-GI-NEXT: mov v5.b[9], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #40]
+; CHECK-GI-NEXT: ldr w12, [sp, #1056]
; CHECK-GI-NEXT: mov v3.b[10], w8
; CHECK-GI-NEXT: ldr w8, [sp, #168]
-; CHECK-GI-NEXT: ldr w10, [sp, #672]
-; CHECK-GI-NEXT: mov v4.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #296]
-; CHECK-GI-NEXT: mov v0.b[11], w11
+; CHECK-GI-NEXT: mov v2.b[10], w11
+; CHECK-GI-NEXT: mov v0.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #808]
+; CHECK-GI-NEXT: ldr w11, [sp, #296]
+; CHECK-GI-NEXT: mov v4.b[10], w10
; CHECK-GI-NEXT: mov v1.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #424]
-; CHECK-GI-NEXT: ldr w11, [sp, #48]
-; CHECK-GI-NEXT: mov v2.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #552]
-; CHECK-GI-NEXT: mov v5.b[10], w10
-; CHECK-GI-NEXT: mov v3.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #176]
-; CHECK-GI-NEXT: ldr w10, [sp, #680]
-; CHECK-GI-NEXT: mov v4.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #304]
-; CHECK-GI-NEXT: mov v0.b[12], w11
-; CHECK-GI-NEXT: mov v1.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #432]
-; CHECK-GI-NEXT: ldr w11, [sp, #56]
-; CHECK-GI-NEXT: mov v2.b[12], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #560]
-; CHECK-GI-NEXT: mov v5.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #936]
+; CHECK-GI-NEXT: mov v5.b[10], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #48]
+; CHECK-GI-NEXT: ldr w12, [sp, #1064]
+; CHECK-GI-NEXT: mov v3.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #176]
+; CHECK-GI-NEXT: mov v2.b[11], w11
+; CHECK-GI-NEXT: mov v0.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #816]
+; CHECK-GI-NEXT: ldr w11, [sp, #304]
+; CHECK-GI-NEXT: mov v4.b[11], w10
+; CHECK-GI-NEXT: mov v1.b[12], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #944]
+; CHECK-GI-NEXT: mov v5.b[11], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #56]
+; CHECK-GI-NEXT: ldr w12, [sp, #1072]
; CHECK-GI-NEXT: mov v3.b[12], w8
; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: ldr w10, [sp, #688]
-; CHECK-GI-NEXT: mov v4.b[12], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #312]
-; CHECK-GI-NEXT: mov v0.b[13], w11
-; CHECK-GI-NEXT: mov v1.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #440]
-; CHECK-GI-NEXT: ldr w11, [sp, #64]
-; CHECK-GI-NEXT: mov v2.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #568]
-; CHECK-GI-NEXT: mov v5.b[12], w10
-; CHECK-GI-NEXT: mov v3.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #192]
-; CHECK-GI-NEXT: ldr w10, [sp, #696]
-; CHECK-GI-NEXT: mov v4.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #320]
-; CHECK-GI-NEXT: mov v0.b[14], w11
-; CHECK-GI-NEXT: mov v1.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #448]
-; CHECK-GI-NEXT: ldr w11, [sp, #72]
-; CHECK-GI-NEXT: mov v2.b[14], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #576]
-; CHECK-GI-NEXT: mov v5.b[13], w10
-; CHECK-GI-NEXT: mov v3.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #720]
-; CHECK-GI-NEXT: ldr w10, [sp, #704]
-; CHECK-GI-NEXT: mov v4.b[14], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #728]
-; CHECK-GI-NEXT: mov v0.b[15], w11
-; CHECK-GI-NEXT: fmov s6, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #328]
-; CHECK-GI-NEXT: ldr w11, [sp, #456]
-; CHECK-GI-NEXT: mov v5.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #200]
-; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[15], w8
-; CHECK-GI-NEXT: mov v3.b[15], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #736]
-; CHECK-GI-NEXT: mov v6.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #584]
-; CHECK-GI-NEXT: ldr w8, [sp, #856]
-; CHECK-GI-NEXT: mov v1.b[15], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #712]
-; CHECK-GI-NEXT: mov v4.b[15], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #976]
-; CHECK-GI-NEXT: mov v7.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1232]
-; CHECK-GI-NEXT: mov v5.b[15], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #984]
-; CHECK-GI-NEXT: mov v6.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1104]
-; CHECK-GI-NEXT: fmov s16, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1360]
-; CHECK-GI-NEXT: fmov s18, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1368]
-; CHECK-GI-NEXT: fmov s17, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1240]
-; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b
-; CHECK-GI-NEXT: mov v16.b[1], w10
-; CHECK-GI-NEXT: fmov s19, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #864]
-; CHECK-GI-NEXT: mov v18.b[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #992]
-; CHECK-GI-NEXT: ldr w9, [sp, #1120]
-; CHECK-GI-NEXT: mov v17.b[1], w12
-; CHECK-GI-NEXT: mov v7.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1248]
-; CHECK-GI-NEXT: mov v19.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #744]
-; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b
-; CHECK-GI-NEXT: mov v16.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #872]
-; CHECK-GI-NEXT: addv s0, v20.4s
-; CHECK-GI-NEXT: mov v6.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1000]
-; CHECK-GI-NEXT: mov v18.b[2], w10
-; CHECK-GI-NEXT: mov v17.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1376]
-; CHECK-GI-NEXT: ldr w10, [sp, #1128]
-; CHECK-GI-NEXT: mov v7.b[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #880]
-; CHECK-GI-NEXT: addv s1, v21.4s
-; CHECK-GI-NEXT: mov v19.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #752]
-; CHECK-GI-NEXT: mov v16.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1256]
-; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b
-; CHECK-GI-NEXT: mov v17.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1384]
-; CHECK-GI-NEXT: mov v6.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1008]
-; CHECK-GI-NEXT: mov v18.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1136]
-; CHECK-GI-NEXT: mov v19.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #760]
-; CHECK-GI-NEXT: mov v7.b[4], w11
-; CHECK-GI-NEXT: mov v16.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1264]
-; CHECK-GI-NEXT: ldr w11, [sp, #888]
-; CHECK-GI-NEXT: mov v17.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1392]
-; CHECK-GI-NEXT: mov v6.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1016]
-; CHECK-GI-NEXT: mov v18.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1144]
-; CHECK-GI-NEXT: mov v19.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #768]
-; CHECK-GI-NEXT: mov v7.b[5], w11
-; CHECK-GI-NEXT: mov v16.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1272]
-; CHECK-GI-NEXT: ldr w11, [sp, #896]
-; CHECK-GI-NEXT: mov v17.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1400]
-; CHECK-GI-NEXT: mov v6.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1024]
-; CHECK-GI-NEXT: mov v18.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1152]
-; CHECK-GI-NEXT: mov v19.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #776]
-; CHECK-GI-NEXT: mov v7.b[6], w11
-; CHECK-GI-NEXT: mov v16.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1280]
-; CHECK-GI-NEXT: ldr w11, [sp, #904]
-; CHECK-GI-NEXT: mov v17.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1408]
-; CHECK-GI-NEXT: mov v6.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1032]
-; CHECK-GI-NEXT: mov v18.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1160]
-; CHECK-GI-NEXT: mov v19.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #784]
-; CHECK-GI-NEXT: mov v7.b[7], w11
-; CHECK-GI-NEXT: mov v16.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1288]
-; CHECK-GI-NEXT: ldr w11, [sp, #912]
-; CHECK-GI-NEXT: mov v17.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1416]
-; CHECK-GI-NEXT: mov v6.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1040]
-; CHECK-GI-NEXT: mov v18.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1168]
-; CHECK-GI-NEXT: mov v19.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #792]
-; CHECK-GI-NEXT: mov v7.b[8], w11
-; CHECK-GI-NEXT: mov v16.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1296]
-; CHECK-GI-NEXT: ldr w11, [sp, #920]
-; CHECK-GI-NEXT: mov v17.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1424]
-; CHECK-GI-NEXT: mov v6.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1048]
-; CHECK-GI-NEXT: mov v18.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1176]
-; CHECK-GI-NEXT: mov v19.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #800]
-; CHECK-GI-NEXT: mov v7.b[9], w11
-; CHECK-GI-NEXT: mov v16.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1304]
-; CHECK-GI-NEXT: ldr w11, [sp, #928]
-; CHECK-GI-NEXT: mov v17.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1432]
-; CHECK-GI-NEXT: mov v6.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1056]
-; CHECK-GI-NEXT: mov v18.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1184]
-; CHECK-GI-NEXT: mov v19.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #808]
-; CHECK-GI-NEXT: mov v7.b[10], w11
-; CHECK-GI-NEXT: mov v16.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1312]
-; CHECK-GI-NEXT: ldr w11, [sp, #936]
-; CHECK-GI-NEXT: mov v17.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1440]
-; CHECK-GI-NEXT: mov v6.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1064]
-; CHECK-GI-NEXT: mov v18.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1192]
-; CHECK-GI-NEXT: mov v19.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #816]
-; CHECK-GI-NEXT: mov v7.b[11], w11
-; CHECK-GI-NEXT: mov v16.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1320]
-; CHECK-GI-NEXT: ldr w11, [sp, #944]
-; CHECK-GI-NEXT: mov v17.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1448]
-; CHECK-GI-NEXT: mov v6.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1072]
-; CHECK-GI-NEXT: mov v18.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1200]
-; CHECK-GI-NEXT: mov v19.b[11], w9
+; CHECK-GI-NEXT: mov v2.b[12], w11
+; CHECK-GI-NEXT: mov v0.b[13], w9
; CHECK-GI-NEXT: ldr w9, [sp, #824]
-; CHECK-GI-NEXT: mov v7.b[12], w11
-; CHECK-GI-NEXT: mov v16.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1328]
-; CHECK-GI-NEXT: ldr w11, [sp, #952]
-; CHECK-GI-NEXT: mov v17.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1456]
-; CHECK-GI-NEXT: mov v6.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1080]
-; CHECK-GI-NEXT: mov v18.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1208]
-; CHECK-GI-NEXT: mov v19.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #832]
-; CHECK-GI-NEXT: mov v7.b[13], w11
-; CHECK-GI-NEXT: mov v16.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1336]
-; CHECK-GI-NEXT: ldr w11, [sp, #960]
-; CHECK-GI-NEXT: mov v17.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1464]
-; CHECK-GI-NEXT: mov v6.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1088]
-; CHECK-GI-NEXT: mov v18.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1216]
-; CHECK-GI-NEXT: mov v19.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #840]
-; CHECK-GI-NEXT: mov v7.b[14], w11
-; CHECK-GI-NEXT: mov v16.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1344]
-; CHECK-GI-NEXT: ldr w11, [sp, #968]
-; CHECK-GI-NEXT: mov v17.b[14], w9
-; CHECK-GI-NEXT: mov v6.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1096]
-; CHECK-GI-NEXT: mov v18.b[14], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #1472]
-; CHECK-GI-NEXT: ldr w10, [sp, #1224]
-; CHECK-GI-NEXT: mov v7.b[15], w11
-; CHECK-GI-NEXT: addv s4, v25.4s
-; CHECK-GI-NEXT: mov v16.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1352]
-; CHECK-GI-NEXT: mov v19.b[14], w9
-; CHECK-GI-NEXT: mov v17.b[15], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #1480]
-; CHECK-GI-NEXT: mov v18.b[15], w8
+; CHECK-GI-NEXT: ldr w11, [sp, #312]
+; CHECK-GI-NEXT: mov v4.b[12], w10
+; CHECK-GI-NEXT: mov v1.b[13], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #952]
+; CHECK-GI-NEXT: mov v5.b[12], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #64]
+; CHECK-GI-NEXT: ldr w12, [sp, #1080]
+; CHECK-GI-NEXT: mov v3.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #192]
+; CHECK-GI-NEXT: mov v2.b[13], w11
+; CHECK-GI-NEXT: mov v0.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #832]
+; CHECK-GI-NEXT: ldr w11, [sp, #320]
+; CHECK-GI-NEXT: mov v4.b[13], w10
+; CHECK-GI-NEXT: mov v1.b[14], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #960]
+; CHECK-GI-NEXT: mov v5.b[13], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #72]
+; CHECK-GI-NEXT: ldr w12, [sp, #1088]
+; CHECK-GI-NEXT: mov v3.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #200]
+; CHECK-GI-NEXT: mov v2.b[14], w11
+; CHECK-GI-NEXT: mov v0.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #840]
+; CHECK-GI-NEXT: ldr w11, [sp, #328]
+; CHECK-GI-NEXT: mov v4.b[14], w10
+; CHECK-GI-NEXT: mov v1.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #968]
+; CHECK-GI-NEXT: mov v5.b[14], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #1096]
+; CHECK-GI-NEXT: mov v3.b[15], w9
+; CHECK-GI-NEXT: mov v2.b[15], w11
+; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b
+; CHECK-GI-NEXT: mov v4.b[15], w8
+; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b
+; CHECK-GI-NEXT: mov v5.b[15], w10
+; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b
+; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b
+; CHECK-GI-NEXT: addv s0, v7.4s
+; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b
+; CHECK-GI-NEXT: addv s1, v16.4s
+; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b
+; CHECK-GI-NEXT: addv s2, v17.4s
+; CHECK-GI-NEXT: addv s4, v20.4s
; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w11, s4
-; CHECK-GI-NEXT: mov v19.b[15], w9
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b
-; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b
+; CHECK-GI-NEXT: addv s3, v18.4s
+; CHECK-GI-NEXT: addv s5, v19.4s
+; CHECK-GI-NEXT: fmov w10, s2
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
-; CHECK-GI-NEXT: add w8, w8, w11
-; CHECK-GI-NEXT: addv s2, v22.4s
-; CHECK-GI-NEXT: addv s3, v23.4s
-; CHECK-GI-NEXT: addv s5, v24.4s
-; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: fmov w10, s3
-; CHECK-GI-NEXT: add w9, w9, w10
-; CHECK-GI-NEXT: fmov w10, s5
-; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: fmov w9, s4
+; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: fmov w11, s5
+; CHECK-GI-NEXT: add w9, w10, w11
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%az = sext <48 x i8> %a to <48 x i32>
- %bz = sext <48 x i8> %b to <48 x i32>
- %m1 = mul nuw nsw <48 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
+ %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
%cz = sext <48 x i8> %c to <48 x i32>
- %dz = sext <48 x i8> %d to <48 x i32>
- %m2 = mul nuw nsw <48 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
+ %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
-define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
+define i32 @test_usdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v48i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q3, [x0, #32]
+; CHECK-SD-NEXT: ldp q7, q2, [x0]
+; CHECK-SD-NEXT: ldr q4, [x1]
+; CHECK-SD-NEXT: ldr q5, [x1, #32]
+; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT: usdot v0.4s, v3.16b, v5.16b
+; CHECK-SD-NEXT: usdot v1.4s, v7.16b, v4.16b
+; CHECK-SD-NEXT: ldr q3, [x1, #16]
+; CHECK-SD-NEXT: usdot v6.4s, v2.16b, v3.16b
+; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v6.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v48i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q0, q2, [x1]
+; CHECK-GI-NEXT: ldr q7, [x1, #32]
+; CHECK-GI-NEXT: ldp q1, q4, [x0]
+; CHECK-GI-NEXT: ldr q6, [x0, #32]
+; CHECK-GI-NEXT: sshll v20.8h, v7.8b, #0
+; CHECK-GI-NEXT: sshll v3.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v18.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v19.8h, v2.8b, #0
+; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v0.8h, v7.16b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v7.8h, v4.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-NEXT: sshll v16.4s, v3.4h, #0
+; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT: sshll v21.4s, v19.4h, #0
+; CHECK-GI-NEXT: ushll v17.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: ushll v25.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0
+; CHECK-GI-NEXT: sshll v22.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: ushll v27.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT: mul v3.4s, v3.4s, v5.4s
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: sshll v17.4s, v18.4h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
+; CHECK-GI-NEXT: ushll v5.8h, v6.8b, #0
+; CHECK-GI-NEXT: ushll2 v6.8h, v6.16b, #0
+; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0
+; CHECK-GI-NEXT: sshll v23.4s, v20.4h, #0
+; CHECK-GI-NEXT: mul v2.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT: sshll v24.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0
+; CHECK-GI-NEXT: addv s3, v3.4s
+; CHECK-GI-NEXT: addv s16, v16.4s
+; CHECK-GI-NEXT: mul v1.4s, v18.4s, v1.4s
+; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0
+; CHECK-GI-NEXT: ushll v18.4s, v6.4h, #0
+; CHECK-GI-NEXT: addv s2, v2.4s
+; CHECK-GI-NEXT: fmov w9, s3
+; CHECK-GI-NEXT: ushll2 v3.4s, v7.8h, #0
+; CHECK-GI-NEXT: mul v7.4s, v17.4s, v25.4s
+; CHECK-GI-NEXT: mul v17.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT: fmov w8, s16
+; CHECK-GI-NEXT: ushll v16.4s, v5.4h, #0
+; CHECK-GI-NEXT: mul v21.4s, v22.4s, v27.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: mul v3.4s, v19.4s, v3.4s
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v4.4s
+; CHECK-GI-NEXT: fmov w12, s2
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: mul v6.4s, v23.4s, v16.4s
+; CHECK-GI-NEXT: mul v16.4s, v24.4s, v18.4s
+; CHECK-GI-NEXT: addv s17, v17.4s
+; CHECK-GI-NEXT: fmov w10, s1
+; CHECK-GI-NEXT: mul v5.4s, v20.4s, v5.4s
+; CHECK-GI-NEXT: addv s4, v21.4s
+; CHECK-GI-NEXT: addv s7, v7.4s
+; CHECK-GI-NEXT: addv s1, v3.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w11, s17
+; CHECK-GI-NEXT: addv s3, v6.4s
+; CHECK-GI-NEXT: fmov w9, s7
+; CHECK-GI-NEXT: addv s2, v5.4s
+; CHECK-GI-NEXT: fmov w13, s1
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: fmov w11, s4
+; CHECK-GI-NEXT: addv s4, v16.4s
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w9, w10, w13
+; CHECK-GI-NEXT: fmov w13, s0
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w12, s3
+; CHECK-GI-NEXT: add w10, w11, w12
+; CHECK-GI-NEXT: fmov w11, s2
+; CHECK-GI-NEXT: fmov w12, s4
+; CHECK-GI-NEXT: add w9, w10, w11
+; CHECK-GI-NEXT: add w10, w12, w13
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w9, w10, w2
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <48 x i8>, ptr %a
+ %1 = zext <48 x i8> %0 to <48 x i32>
+ %2 = load <48 x i8>, ptr %b
+ %3 = sext <48 x i8> %2 to <48 x i32>
+ %4 = mul nsw <48 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v48i8_double:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: .cfi_offset w29, -16
+; CHECK-SD-NEXT: ldr b2, [sp, #592]
+; CHECK-SD-NEXT: add x8, sp, #600
; CHECK-SD-NEXT: ldr b5, [sp, #208]
+; CHECK-SD-NEXT: add x9, sp, #616
+; CHECK-SD-NEXT: ldr b0, [sp, #336]
+; CHECK-SD-NEXT: add x10, sp, #624
+; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #216
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: ldr b4, [sp, #976]
-; CHECK-SD-NEXT: add x9, sp, #984
-; CHECK-SD-NEXT: add x12, sp, #328
+; CHECK-SD-NEXT: add x11, sp, #240
; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #608
+; CHECK-SD-NEXT: ldr b4, [sp, #464]
+; CHECK-SD-NEXT: ldr b6, [sp, #80]
+; CHECK-SD-NEXT: ldr b7, [sp, #1360]
+; CHECK-SD-NEXT: ldr b16, [sp, #976]
+; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #224
-; CHECK-SD-NEXT: movi v1.16b, #1
-; CHECK-SD-NEXT: mov v0.b[1], w1
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9]
-; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT: add x11, sp, #992
-; CHECK-SD-NEXT: ldr b6, [sp, #720]
-; CHECK-SD-NEXT: ldr b7, [sp, #80]
+; CHECK-SD-NEXT: fmov s1, w0
; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #344
+; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #232
-; CHECK-SD-NEXT: add x13, sp, #88
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
-; CHECK-SD-NEXT: add x13, sp, #856
-; CHECK-SD-NEXT: mov v0.b[2], w2
-; CHECK-SD-NEXT: add x14, sp, #1008
-; CHECK-SD-NEXT: add x15, sp, #872
+; CHECK-SD-NEXT: ldr b18, [sp, #720]
+; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #632
+; CHECK-SD-NEXT: mov v1.b[1], w1
; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #240
-; CHECK-SD-NEXT: add x16, sp, #888
-; CHECK-SD-NEXT: add x10, sp, #16
-; CHECK-SD-NEXT: add x9, sp, #24
-; CHECK-SD-NEXT: add x11, sp, #40
-; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #248
-; CHECK-SD-NEXT: mov v0.b[3], w3
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #256
-; CHECK-SD-NEXT: mov v0.b[4], w4
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
-; CHECK-SD-NEXT: add x8, sp, #264
-; CHECK-SD-NEXT: mov v0.b[5], w5
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #272
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8]
-; CHECK-SD-NEXT: add x8, sp, #280
-; CHECK-SD-NEXT: mov v0.b[6], w6
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8]
-; CHECK-SD-NEXT: add x8, sp, #288
-; CHECK-SD-NEXT: mov v0.b[7], w7
-; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #296
-; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10]
-; CHECK-SD-NEXT: add x10, sp, #128
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #304
-; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9]
-; CHECK-SD-NEXT: add x9, sp, #136
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #312
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #320
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT: add x8, sp, #32
-; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #144
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12]
-; CHECK-SD-NEXT: add x12, sp, #728
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1000
-; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12]
-; CHECK-SD-NEXT: add x12, sp, #736
-; CHECK-SD-NEXT: add x11, sp, #920
-; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b
-; CHECK-SD-NEXT: ldr b5, [sp, #848]
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
-; CHECK-SD-NEXT: add x12, sp, #48
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13]
-; CHECK-SD-NEXT: add x13, sp, #744
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14]
-; CHECK-SD-NEXT: add x14, sp, #96
-; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13]
+; CHECK-SD-NEXT: add x8, sp, #472
+; CHECK-SD-NEXT: add x12, sp, #856
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #640
; CHECK-SD-NEXT: add x13, sp, #864
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14]
-; CHECK-SD-NEXT: add x14, sp, #1016
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT: add x13, sp, #752
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14]
-; CHECK-SD-NEXT: add x14, sp, #104
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
-; CHECK-SD-NEXT: add x13, sp, #1024
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14]
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15]
-; CHECK-SD-NEXT: add x15, sp, #760
-; CHECK-SD-NEXT: add x14, sp, #112
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
-; CHECK-SD-NEXT: add x13, sp, #880
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15]
-; CHECK-SD-NEXT: add x15, sp, #1032
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
-; CHECK-SD-NEXT: add x14, sp, #768
-; CHECK-SD-NEXT: add x13, sp, #120
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15]
-; CHECK-SD-NEXT: add x15, sp, #1040
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14]
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
-; CHECK-SD-NEXT: add x13, sp, #776
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16]
-; CHECK-SD-NEXT: add x14, sp, #1048
-; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15]
-; CHECK-SD-NEXT: add x15, sp, #896
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13]
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #784
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15]
-; CHECK-SD-NEXT: add x13, sp, #1056
-; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
-; CHECK-SD-NEXT: add x14, sp, #904
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #792
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14]
-; CHECK-SD-NEXT: add x10, sp, #1064
-; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13]
-; CHECK-SD-NEXT: add x13, sp, #912
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
-; CHECK-SD-NEXT: add x9, sp, #800
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13]
-; CHECK-SD-NEXT: add x8, sp, #152
-; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1072
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9]
-; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
-; CHECK-SD-NEXT: add x9, sp, #808
+; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #88
+; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x10]
+; CHECK-SD-NEXT: add x10, sp, #248
+; CHECK-SD-NEXT: add x11, sp, #272
+; CHECK-SD-NEXT: mov v1.b[2], w2
+; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1368
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #984
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #352
+; CHECK-SD-NEXT: add x10, sp, #648
+; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v0.b }[2], [x9]
+; CHECK-SD-NEXT: add x9, sp, #256
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
+; CHECK-SD-NEXT: add x8, sp, #480
+; CHECK-SD-NEXT: add x9, sp, #96
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #656
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10]
+; CHECK-SD-NEXT: add x9, sp, #264
+; CHECK-SD-NEXT: add x10, sp, #1376
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #664
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #992
+; CHECK-SD-NEXT: mov v1.b[3], w3
+; CHECK-SD-NEXT: ld1 { v2.b }[8], [x8]
+; CHECK-SD-NEXT: add x8, sp, #360
+; CHECK-SD-NEXT: ld1 { v16.b }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11]
+; CHECK-SD-NEXT: add x11, sp, #488
+; CHECK-SD-NEXT: add x10, sp, #672
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #280
+; CHECK-SD-NEXT: ld1 { v2.b }[9], [x9]
+; CHECK-SD-NEXT: add x9, sp, #104
+; CHECK-SD-NEXT: add x8, sp, #680
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #368
; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
-; CHECK-SD-NEXT: add x8, sp, #56
-; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10]
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1384
+; CHECK-SD-NEXT: add x11, sp, #688
+; CHECK-SD-NEXT: ld1 { v2.b }[10], [x10]
+; CHECK-SD-NEXT: add x10, sp, #376
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #288
+; CHECK-SD-NEXT: mov v1.b[4], w4
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
+; CHECK-SD-NEXT: add x10, sp, #296
+; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #496
+; CHECK-SD-NEXT: add x9, sp, #1000
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #384
+; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x10]
+; CHECK-SD-NEXT: add x8, sp, #112
+; CHECK-SD-NEXT: ld1 { v2.b }[12], [x11]
+; CHECK-SD-NEXT: add x11, sp, #392
+; CHECK-SD-NEXT: add x9, sp, #696
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #304
+; CHECK-SD-NEXT: add x10, sp, #704
+; CHECK-SD-NEXT: ld1 { v0.b }[7], [x11]
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT: add x11, sp, #312
+; CHECK-SD-NEXT: ld1 { v2.b }[13], [x9]
+; CHECK-SD-NEXT: add x9, sp, #504
+; CHECK-SD-NEXT: add x8, sp, #1392
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #400
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x8]
+; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x11]
+; CHECK-SD-NEXT: add x11, sp, #320
+; CHECK-SD-NEXT: ld1 { v2.b }[14], [x10]
+; CHECK-SD-NEXT: add x10, sp, #120
+; CHECK-SD-NEXT: add x9, sp, #328
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #408
+; CHECK-SD-NEXT: add x8, sp, #712
+; CHECK-SD-NEXT: ld1 { v0.b }[9], [x10]
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x11]
+; CHECK-SD-NEXT: add x11, sp, #416
+; CHECK-SD-NEXT: add x10, sp, #512
+; CHECK-SD-NEXT: ld1 { v2.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #128
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1016
+; CHECK-SD-NEXT: ld1 { v0.b }[10], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1400
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1008
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1408
+; CHECK-SD-NEXT: ld1 { v16.b }[4], [x9]
+; CHECK-SD-NEXT: add x9, sp, #520
+; CHECK-SD-NEXT: add x10, sp, #424
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v0.b }[11], [x10]
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
+; CHECK-SD-NEXT: add x10, sp, #1416
+; CHECK-SD-NEXT: ld1 { v16.b }[5], [x8]
+; CHECK-SD-NEXT: add x8, sp, #528
+; CHECK-SD-NEXT: add x9, sp, #144
+; CHECK-SD-NEXT: ld1 { v4.b }[8], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1024
+; CHECK-SD-NEXT: add x11, sp, #1432
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x9]
+; CHECK-SD-NEXT: add x10, sp, #1424
+; CHECK-SD-NEXT: ld1 { v16.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #536
+; CHECK-SD-NEXT: add x9, sp, #152
+; CHECK-SD-NEXT: ld1 { v4.b }[9], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1032
+; CHECK-SD-NEXT: usdot v3.4s, v5.16b, v2.16b
+; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
; CHECK-SD-NEXT: add x10, sp, #160
+; CHECK-SD-NEXT: ld1 { v16.b }[7], [x8]
+; CHECK-SD-NEXT: add x8, sp, #544
+; CHECK-SD-NEXT: add x9, sp, #432
+; CHECK-SD-NEXT: ld1 { v4.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1040
+; CHECK-SD-NEXT: ld1 { v0.b }[12], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[9], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x10]
+; CHECK-SD-NEXT: add x11, sp, #1440
+; CHECK-SD-NEXT: ld1 { v16.b }[8], [x8]
+; CHECK-SD-NEXT: add x8, sp, #552
+; CHECK-SD-NEXT: add x10, sp, #168
+; CHECK-SD-NEXT: ld1 { v4.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1048
+; CHECK-SD-NEXT: add x9, sp, #176
+; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1448
+; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8]
+; CHECK-SD-NEXT: add x8, sp, #560
+; CHECK-SD-NEXT: add x11, sp, #576
+; CHECK-SD-NEXT: ld1 { v4.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1056
+; CHECK-SD-NEXT: ldr b5, [sp, #1104]
+; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9]
+; CHECK-SD-NEXT: add x10, sp, #1456
+; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #568
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: ld1 { v4.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1064
+; CHECK-SD-NEXT: mov v1.b[5], w5
+; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1072
+; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8]
+; CHECK-SD-NEXT: add x10, sp, #192
+; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v4.b }[14], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1464
+; CHECK-SD-NEXT: add x8, sp, #440
+; CHECK-SD-NEXT: ld1 { v7.b }[13], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10]
+; CHECK-SD-NEXT: add x11, sp, #1472
+; CHECK-SD-NEXT: ld1 { v16.b }[12], [x9]
+; CHECK-SD-NEXT: add x9, sp, #584
+; CHECK-SD-NEXT: add x10, sp, #200
+; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1080
+; CHECK-SD-NEXT: mov v1.b[6], w6
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1112
+; CHECK-SD-NEXT: ld1 { v16.b }[13], [x9]
+; CHECK-SD-NEXT: add x11, sp, #1480
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
+; CHECK-SD-NEXT: add x9, sp, #1088
+; CHECK-SD-NEXT: add x10, sp, #1120
; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9]
-; CHECK-SD-NEXT: add x9, sp, #928
-; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1080
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x11]
+; CHECK-SD-NEXT: add x11, sp, #728
+; CHECK-SD-NEXT: usdot v2.4s, v6.16b, v4.16b
+; CHECK-SD-NEXT: ld1 { v18.b }[1], [x11]
+; CHECK-SD-NEXT: ld1 { v16.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #736
+; CHECK-SD-NEXT: add x9, sp, #1096
+; CHECK-SD-NEXT: ldr b4, [sp, #1232]
+; CHECK-SD-NEXT: ldr b6, [sp, #848]
+; CHECK-SD-NEXT: add x11, sp, #1240
+; CHECK-SD-NEXT: mov v1.b[7], w7
+; CHECK-SD-NEXT: ld1 { v18.b }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1128
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x9]
+; CHECK-SD-NEXT: add x10, sp, #744
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1136
+; CHECK-SD-NEXT: add x12, sp, #1248
+; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10]
+; CHECK-SD-NEXT: add x11, sp, #752
+; CHECK-SD-NEXT: add x10, sp, #16
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1144
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x13]
+; CHECK-SD-NEXT: add x12, sp, #872
+; CHECK-SD-NEXT: ld1 { v1.b }[8], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1256
+; CHECK-SD-NEXT: add x10, sp, #760
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11]
+; CHECK-SD-NEXT: add x9, sp, #1152
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1264
+; CHECK-SD-NEXT: add x13, sp, #880
+; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10]
+; CHECK-SD-NEXT: add x11, sp, #768
+; CHECK-SD-NEXT: add x10, sp, #24
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1160
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
+; CHECK-SD-NEXT: add x12, sp, #888
+; CHECK-SD-NEXT: ld1 { v1.b }[9], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1272
+; CHECK-SD-NEXT: add x10, sp, #776
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x11]
+; CHECK-SD-NEXT: add x9, sp, #1168
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1280
+; CHECK-SD-NEXT: add x13, sp, #896
+; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10]
+; CHECK-SD-NEXT: add x11, sp, #784
+; CHECK-SD-NEXT: add x10, sp, #32
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1176
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x13]
+; CHECK-SD-NEXT: add x12, sp, #904
+; CHECK-SD-NEXT: ld1 { v1.b }[10], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[8], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1288
+; CHECK-SD-NEXT: add x10, sp, #792
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x11]
+; CHECK-SD-NEXT: add x9, sp, #1184
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1296
+; CHECK-SD-NEXT: add x13, sp, #912
+; CHECK-SD-NEXT: ld1 { v18.b }[9], [x10]
+; CHECK-SD-NEXT: add x11, sp, #800
+; CHECK-SD-NEXT: add x10, sp, #40
; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
-; CHECK-SD-NEXT: add x8, sp, #816
-; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
-; CHECK-SD-NEXT: add x9, sp, #168
-; CHECK-SD-NEXT: add x10, sp, #176
-; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #936
-; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1088
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #64
-; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #824
-; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
-; CHECK-SD-NEXT: add x9, sp, #944
-; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1096
+; CHECK-SD-NEXT: ld1 { v4.b }[8], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1192
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x13]
+; CHECK-SD-NEXT: add x12, sp, #920
+; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[10], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1304
+; CHECK-SD-NEXT: add x10, sp, #808
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[9], [x11]
+; CHECK-SD-NEXT: add x9, sp, #1200
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1312
+; CHECK-SD-NEXT: add x13, sp, #928
+; CHECK-SD-NEXT: ld1 { v18.b }[11], [x10]
+; CHECK-SD-NEXT: add x11, sp, #816
+; CHECK-SD-NEXT: add x10, sp, #48
; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
-; CHECK-SD-NEXT: add x8, sp, #832
-; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10]
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: add x10, sp, #72
-; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
-; CHECK-SD-NEXT: add x8, sp, #952
-; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #840
-; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
-; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b
-; CHECK-SD-NEXT: add x9, sp, #192
-; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #960
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
-; CHECK-SD-NEXT: add x8, sp, #200
-; CHECK-SD-NEXT: add x9, sp, #968
-; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v4.b }[10], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1208
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x13]
+; CHECK-SD-NEXT: add x12, sp, #936
+; CHECK-SD-NEXT: ld1 { v1.b }[12], [x10]
+; CHECK-SD-NEXT: ld1 { v18.b }[12], [x11]
+; CHECK-SD-NEXT: add x11, sp, #1320
+; CHECK-SD-NEXT: add x10, sp, #824
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[11], [x11]
+; CHECK-SD-NEXT: add x9, sp, #1216
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x12]
+; CHECK-SD-NEXT: add x12, sp, #1328
+; CHECK-SD-NEXT: add x13, sp, #944
+; CHECK-SD-NEXT: ld1 { v18.b }[13], [x10]
+; CHECK-SD-NEXT: add x11, sp, #832
+; CHECK-SD-NEXT: add x10, sp, #56
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v4.b }[12], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1224
+; CHECK-SD-NEXT: ld1 { v6.b }[12], [x13]
+; CHECK-SD-NEXT: ld1 { v1.b }[13], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1336
+; CHECK-SD-NEXT: ld1 { v18.b }[14], [x11]
+; CHECK-SD-NEXT: add x11, sp, #952
+; CHECK-SD-NEXT: add x8, sp, #448
; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
-; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b
-; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT: add x9, sp, #840
+; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11]
+; CHECK-SD-NEXT: usdot v17.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v18.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #64
+; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1344
+; CHECK-SD-NEXT: add x10, sp, #960
+; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10]
+; CHECK-SD-NEXT: add x8, sp, #456
+; CHECK-SD-NEXT: usdot v7.4s, v18.16b, v5.16b
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v0.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #72
+; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
+; CHECK-SD-NEXT: add x9, sp, #1352
+; CHECK-SD-NEXT: add x10, sp, #968
+; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10]
+; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v0.16b
; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: add v1.4s, v7.4s, v17.4s
+; CHECK-SD-NEXT: usdot v5.4s, v6.16b, v4.16b
+; CHECK-SD-NEXT: add v0.4s, v16.4s, v0.4s
+; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT: ret
;
-; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
+; CHECK-GI-LABEL: test_usdot_v48i8_double:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w10, [sp, #80]
-; CHECK-GI-NEXT: ldr w11, [sp, #208]
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w9, [sp, #88]
-; CHECK-GI-NEXT: ldr w12, [sp, #728]
-; CHECK-GI-NEXT: movi v6.16b, #1
-; CHECK-GI-NEXT: fmov s1, w10
-; CHECK-GI-NEXT: fmov s2, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #720]
-; CHECK-GI-NEXT: ldr w10, [sp, #216]
-; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: ldr w13, [sp, #856]
-; CHECK-GI-NEXT: fmov s3, w11
-; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: ldr w11, [sp, #224]
-; CHECK-GI-NEXT: mov v1.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #848]
-; CHECK-GI-NEXT: mov v2.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #976]
-; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
-; CHECK-GI-NEXT: fmov s4, w9
-; CHECK-GI-NEXT: mov v3.b[1], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #984]
-; CHECK-GI-NEXT: fmov s5, w10
-; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: ldr w10, [sp, #736]
-; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #864]
-; CHECK-GI-NEXT: mov v2.b[2], w11
-; CHECK-GI-NEXT: mov v4.b[1], w13
-; CHECK-GI-NEXT: ldr w11, [sp, #992]
-; CHECK-GI-NEXT: ldr w12, [sp, #776]
-; CHECK-GI-NEXT: mov v5.b[1], w9
-; CHECK-GI-NEXT: mov v3.b[2], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #104]
-; CHECK-GI-NEXT: ldr w10, [sp, #232]
-; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #872]
-; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v4.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #744]
-; CHECK-GI-NEXT: mov v2.b[3], w10
-; CHECK-GI-NEXT: mov v5.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1000]
-; CHECK-GI-NEXT: ldr w10, [sp, #240]
-; CHECK-GI-NEXT: mov v3.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v4.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #752]
-; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #880]
-; CHECK-GI-NEXT: mov v5.b[3], w11
-; CHECK-GI-NEXT: mov v2.b[4], w10
-; CHECK-GI-NEXT: mov v3.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #120]
-; CHECK-GI-NEXT: ldr w11, [sp, #1008]
-; CHECK-GI-NEXT: ldr w10, [sp, #248]
-; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v4.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #760]
-; CHECK-GI-NEXT: mov v1.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #888]
-; CHECK-GI-NEXT: mov v5.b[4], w11
-; CHECK-GI-NEXT: mov v2.b[5], w10
-; CHECK-GI-NEXT: mov v3.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: ldr w11, [sp, #1016]
-; CHECK-GI-NEXT: ldr w10, [sp, #256]
-; CHECK-GI-NEXT: mov v0.b[6], w6
-; CHECK-GI-NEXT: mov v4.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #768]
-; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #896]
-; CHECK-GI-NEXT: mov v5.b[5], w11
-; CHECK-GI-NEXT: mov v2.b[6], w10
-; CHECK-GI-NEXT: mov v3.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #136]
-; CHECK-GI-NEXT: ldr w11, [sp, #1024]
-; CHECK-GI-NEXT: ldr w10, [sp, #264]
-; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v4.b[6], w8
-; CHECK-GI-NEXT: mov v1.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #904]
-; CHECK-GI-NEXT: mov v5.b[6], w11
-; CHECK-GI-NEXT: mov v2.b[7], w10
; CHECK-GI-NEXT: ldr w8, [sp, #16]
-; CHECK-GI-NEXT: mov v3.b[7], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #144]
-; CHECK-GI-NEXT: ldr w12, [sp, #1032]
-; CHECK-GI-NEXT: mov v0.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #784]
-; CHECK-GI-NEXT: ldr w11, [sp, #272]
-; CHECK-GI-NEXT: mov v4.b[7], w9
-; CHECK-GI-NEXT: mov v1.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #912]
-; CHECK-GI-NEXT: mov v5.b[7], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #24]
-; CHECK-GI-NEXT: ldr w12, [sp, #1040]
-; CHECK-GI-NEXT: mov v3.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #152]
-; CHECK-GI-NEXT: mov v2.b[8], w11
-; CHECK-GI-NEXT: mov v0.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #792]
-; CHECK-GI-NEXT: ldr w11, [sp, #280]
-; CHECK-GI-NEXT: mov v4.b[8], w10
-; CHECK-GI-NEXT: mov v1.b[9], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #920]
-; CHECK-GI-NEXT: mov v5.b[8], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #80]
+; CHECK-GI-NEXT: fmov s2, w0
+; CHECK-GI-NEXT: ldr w10, [sp, #24]
+; CHECK-GI-NEXT: ldr w11, [sp, #144]
+; CHECK-GI-NEXT: ldr w12, [sp, #296]
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: fmov s4, w9
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: fmov s5, w11
+; CHECK-GI-NEXT: ldr w9, [sp, #152]
+; CHECK-GI-NEXT: ldr w11, [sp, #216]
+; CHECK-GI-NEXT: ldr w13, [sp, #64]
+; CHECK-GI-NEXT: ldr w14, [sp, #128]
+; CHECK-GI-NEXT: mov v2.b[1], w1
+; CHECK-GI-NEXT: mov v3.b[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #208]
+; CHECK-GI-NEXT: mov v4.b[1], w8
; CHECK-GI-NEXT: ldr w8, [sp, #32]
-; CHECK-GI-NEXT: ldr w12, [sp, #1048]
-; CHECK-GI-NEXT: mov v3.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #160]
-; CHECK-GI-NEXT: mov v2.b[9], w11
-; CHECK-GI-NEXT: mov v0.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #800]
-; CHECK-GI-NEXT: ldr w11, [sp, #288]
-; CHECK-GI-NEXT: mov v4.b[9], w10
-; CHECK-GI-NEXT: mov v1.b[10], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #928]
-; CHECK-GI-NEXT: mov v5.b[9], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #40]
-; CHECK-GI-NEXT: ldr w12, [sp, #1056]
-; CHECK-GI-NEXT: mov v3.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #168]
-; CHECK-GI-NEXT: mov v2.b[10], w11
-; CHECK-GI-NEXT: mov v0.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #808]
-; CHECK-GI-NEXT: ldr w11, [sp, #296]
-; CHECK-GI-NEXT: mov v4.b[10], w10
-; CHECK-GI-NEXT: mov v1.b[11], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #936]
-; CHECK-GI-NEXT: mov v5.b[10], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #48]
-; CHECK-GI-NEXT: ldr w12, [sp, #1064]
-; CHECK-GI-NEXT: mov v3.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #176]
-; CHECK-GI-NEXT: mov v2.b[11], w11
-; CHECK-GI-NEXT: mov v0.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #816]
-; CHECK-GI-NEXT: ldr w11, [sp, #304]
-; CHECK-GI-NEXT: mov v4.b[11], w10
-; CHECK-GI-NEXT: mov v1.b[12], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #944]
-; CHECK-GI-NEXT: mov v5.b[11], w12
+; CHECK-GI-NEXT: mov v5.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #96]
+; CHECK-GI-NEXT: fmov s6, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #224]
+; CHECK-GI-NEXT: ldr w15, [sp, #264]
+; CHECK-GI-NEXT: ldr w16, [sp, #528]
+; CHECK-GI-NEXT: mov v2.b[2], w2
+; CHECK-GI-NEXT: ldr w17, [sp, #648]
+; CHECK-GI-NEXT: mov v3.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #160]
+; CHECK-GI-NEXT: mov v4.b[2], w9
+; CHECK-GI-NEXT: mov v6.b[1], w11
+; CHECK-GI-NEXT: ldr w9, [sp, #272]
+; CHECK-GI-NEXT: ldr w11, [sp, #48]
+; CHECK-GI-NEXT: mov v5.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #40]
+; CHECK-GI-NEXT: fmov s18, w16
+; CHECK-GI-NEXT: fmov s0, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #168]
+; CHECK-GI-NEXT: mov v2.b[3], w3
+; CHECK-GI-NEXT: mov v3.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #232]
+; CHECK-GI-NEXT: ldr w16, [sp, #584]
+; CHECK-GI-NEXT: mov v6.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #104]
+; CHECK-GI-NEXT: mov v5.b[3], w9
; CHECK-GI-NEXT: ldr w9, [sp, #56]
-; CHECK-GI-NEXT: ldr w12, [sp, #1072]
-; CHECK-GI-NEXT: mov v3.b[12], w8
+; CHECK-GI-NEXT: mov v4.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #280]
+; CHECK-GI-NEXT: mov v2.b[4], w4
+; CHECK-GI-NEXT: mov v3.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #240]
+; CHECK-GI-NEXT: mov v6.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #112]
+; CHECK-GI-NEXT: mov v0.b[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #176]
+; CHECK-GI-NEXT: mov v4.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #288]
+; CHECK-GI-NEXT: mov v2.b[5], w5
+; CHECK-GI-NEXT: mov v5.b[4], w10
+; CHECK-GI-NEXT: mov v3.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #248]
+; CHECK-GI-NEXT: mov v0.b[2], w8
+; CHECK-GI-NEXT: mov v6.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #120]
; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: mov v2.b[12], w11
-; CHECK-GI-NEXT: mov v0.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #824]
-; CHECK-GI-NEXT: ldr w11, [sp, #312]
-; CHECK-GI-NEXT: mov v4.b[12], w10
-; CHECK-GI-NEXT: mov v1.b[13], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #952]
-; CHECK-GI-NEXT: mov v5.b[12], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #64]
-; CHECK-GI-NEXT: ldr w12, [sp, #1080]
-; CHECK-GI-NEXT: mov v3.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #192]
-; CHECK-GI-NEXT: mov v2.b[13], w11
-; CHECK-GI-NEXT: mov v0.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #832]
-; CHECK-GI-NEXT: ldr w11, [sp, #320]
-; CHECK-GI-NEXT: mov v4.b[13], w10
-; CHECK-GI-NEXT: mov v1.b[14], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #960]
-; CHECK-GI-NEXT: mov v5.b[13], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #72]
-; CHECK-GI-NEXT: ldr w12, [sp, #1088]
-; CHECK-GI-NEXT: mov v3.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #200]
-; CHECK-GI-NEXT: mov v2.b[14], w11
-; CHECK-GI-NEXT: mov v0.b[15], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #72]
+; CHECK-GI-NEXT: mov v4.b[5], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #136]
+; CHECK-GI-NEXT: mov v2.b[6], w6
+; CHECK-GI-NEXT: mov v5.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #192]
+; CHECK-GI-NEXT: mov v3.b[6], w13
+; CHECK-GI-NEXT: mov v0.b[3], w12
+; CHECK-GI-NEXT: mov v6.b[5], w9
+; CHECK-GI-NEXT: ldr w13, [sp, #304]
+; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: ldr w12, [sp, #200]
+; CHECK-GI-NEXT: mov v4.b[6], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #336]
+; CHECK-GI-NEXT: mov v2.b[7], w7
+; CHECK-GI-NEXT: mov v5.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #344]
+; CHECK-GI-NEXT: mov v3.b[7], w10
+; CHECK-GI-NEXT: mov v0.b[4], w13
+; CHECK-GI-NEXT: fmov s7, w14
+; CHECK-GI-NEXT: mov v6.b[6], w9
+; CHECK-GI-NEXT: ldr w14, [sp, #312]
+; CHECK-GI-NEXT: ldr w13, [sp, #352]
+; CHECK-GI-NEXT: ldr w10, [sp, #360]
+; CHECK-GI-NEXT: mov v4.b[7], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #472]
+; CHECK-GI-NEXT: ldr w9, [sp, #320]
+; CHECK-GI-NEXT: mov v5.b[7], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #400]
+; CHECK-GI-NEXT: mov v7.b[1], w8
+; CHECK-GI-NEXT: mov v0.b[5], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #464]
+; CHECK-GI-NEXT: mov v6.b[7], w15
+; CHECK-GI-NEXT: fmov s16, w12
+; CHECK-GI-NEXT: ldr w15, [sp, #408]
+; CHECK-GI-NEXT: ldr w8, [sp, #328]
+; CHECK-GI-NEXT: fmov s17, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #592]
+; CHECK-GI-NEXT: ldr w12, [sp, #368]
+; CHECK-GI-NEXT: mov v7.b[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #536]
+; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT: mov v16.b[1], w15
+; CHECK-GI-NEXT: fmov s19, w14
+; CHECK-GI-NEXT: ldr w15, [sp, #600]
+; CHECK-GI-NEXT: mov v18.b[1], w13
+; CHECK-GI-NEXT: ldr w14, [sp, #416]
+; CHECK-GI-NEXT: mov v17.b[1], w11
+; CHECK-GI-NEXT: mov v0.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: ldr w11, [sp, #376]
+; CHECK-GI-NEXT: mov v19.b[1], w15
+; CHECK-GI-NEXT: mov v7.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #544]
+; CHECK-GI-NEXT: mov v16.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #608]
+; CHECK-GI-NEXT: ldr w13, [sp, #384]
+; CHECK-GI-NEXT: mov v18.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #424]
+; CHECK-GI-NEXT: mov v17.b[2], w9
+; CHECK-GI-NEXT: mov v0.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #488]
+; CHECK-GI-NEXT: ldr w9, [sp, #392]
+; CHECK-GI-NEXT: mov v19.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #552]
+; CHECK-GI-NEXT: mov v7.b[4], w12
+; CHECK-GI-NEXT: mov v16.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #656]
+; CHECK-GI-NEXT: ldr w12, [sp, #616]
+; CHECK-GI-NEXT: mov v17.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #432]
+; CHECK-GI-NEXT: mov v18.b[3], w14
+; CHECK-GI-NEXT: fmov s1, w10
+; CHECK-GI-NEXT: ldr w14, [sp, #664]
+; CHECK-GI-NEXT: ldr w10, [sp, #440]
+; CHECK-GI-NEXT: mov v19.b[3], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #496]
+; CHECK-GI-NEXT: mov v7.b[5], w11
+; CHECK-GI-NEXT: mov v16.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #624]
+; CHECK-GI-NEXT: ldr w11, [sp, #504]
+; CHECK-GI-NEXT: mov v1.b[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #560]
+; CHECK-GI-NEXT: mov v17.b[4], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #448]
+; CHECK-GI-NEXT: ldr w15, [sp, #456]
+; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: mov v18.b[4], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #672]
+; CHECK-GI-NEXT: mov v19.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #568]
+; CHECK-GI-NEXT: mov v16.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #632]
+; CHECK-GI-NEXT: mov v1.b[2], w14
+; CHECK-GI-NEXT: mov v17.b[5], w11
+; CHECK-GI-NEXT: mov v7.b[6], w13
+; CHECK-GI-NEXT: ldr w11, [sp, #512]
+; CHECK-GI-NEXT: ldr w13, [sp, #520]
+; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT: mov v18.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #680]
+; CHECK-GI-NEXT: mov v19.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #576]
+; CHECK-GI-NEXT: mov v16.b[6], w12
+; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
+; CHECK-GI-NEXT: mov v1.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #640]
+; CHECK-GI-NEXT: mov v17.b[6], w11
+; CHECK-GI-NEXT: mov v7.b[7], w9
+; CHECK-GI-NEXT: ushll v20.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: mov v18.b[6], w10
+; CHECK-GI-NEXT: mov v19.b[6], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #688]
+; CHECK-GI-NEXT: mov v16.b[7], w15
+; CHECK-GI-NEXT: ushll v21.4s, v3.4h, #0
+; CHECK-GI-NEXT: ushll2 v22.4s, v3.8h, #0
+; CHECK-GI-NEXT: mov v17.b[7], w13
+; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll2 v24.4s, v4.8h, #0
+; CHECK-GI-NEXT: sshll v7.8h, v7.8b, #0
+; CHECK-GI-NEXT: ushll v3.4s, v6.4h, #0
+; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0
+; CHECK-GI-NEXT: mov v18.b[7], w16
+; CHECK-GI-NEXT: mov v19.b[7], w17
+; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
+; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT: mov v1.b[4], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #696]
+; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
+; CHECK-GI-NEXT: ldr w14, [sp, #720]
+; CHECK-GI-NEXT: ldr w13, [sp, #728]
+; CHECK-GI-NEXT: ushll v25.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: ldr w12, [sp, #704]
+; CHECK-GI-NEXT: sshll v6.8h, v18.8b, #0
+; CHECK-GI-NEXT: sshll v18.8h, v19.8b, #0
+; CHECK-GI-NEXT: sshll v19.4s, v7.4h, #0
+; CHECK-GI-NEXT: sshll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: sshll v26.4s, v16.4h, #0
+; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT: sshll v27.4s, v17.4h, #0
+; CHECK-GI-NEXT: mov v1.b[5], w9
+; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
+; CHECK-GI-NEXT: mul v19.4s, v20.4s, v19.4s
+; CHECK-GI-NEXT: sshll v28.4s, v6.4h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v6.8h, #0
+; CHECK-GI-NEXT: mul v7.4s, v2.4s, v7.4s
+; CHECK-GI-NEXT: mul v20.4s, v21.4s, v26.4s
+; CHECK-GI-NEXT: mul v16.4s, v22.4s, v16.4s
+; CHECK-GI-NEXT: mul v21.4s, v23.4s, v27.4s
+; CHECK-GI-NEXT: fmov s2, w14
+; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
+; CHECK-GI-NEXT: mul v5.4s, v5.4s, v6.4s
+; CHECK-GI-NEXT: mul v17.4s, v24.4s, v17.4s
+; CHECK-GI-NEXT: addv s6, v19.4s
+; CHECK-GI-NEXT: mov v1.b[6], w12
+; CHECK-GI-NEXT: ldr w11, [sp, #736]
+; CHECK-GI-NEXT: addv s7, v7.4s
+; CHECK-GI-NEXT: mov v2.b[1], w13
+; CHECK-GI-NEXT: addv s16, v16.4s
+; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s
+; CHECK-GI-NEXT: addv s18, v20.4s
+; CHECK-GI-NEXT: addv s19, v21.4s
+; CHECK-GI-NEXT: fmov w12, s6
+; CHECK-GI-NEXT: mul v22.4s, v25.4s, v28.4s
+; CHECK-GI-NEXT: addv s17, v17.4s
+; CHECK-GI-NEXT: fmov w13, s7
+; CHECK-GI-NEXT: mul v3.4s, v3.4s, v29.4s
+; CHECK-GI-NEXT: addv s5, v5.4s
+; CHECK-GI-NEXT: mov v2.b[2], w11
+; CHECK-GI-NEXT: fmov w14, s19
+; CHECK-GI-NEXT: ldr w10, [sp, #744]
+; CHECK-GI-NEXT: ldr w9, [sp, #752]
+; CHECK-GI-NEXT: ldr w8, [sp, #712]
+; CHECK-GI-NEXT: ldr w15, [sp, #864]
+; CHECK-GI-NEXT: add w11, w12, w13
+; CHECK-GI-NEXT: fmov w12, s18
+; CHECK-GI-NEXT: fmov w13, s16
+; CHECK-GI-NEXT: addv s20, v22.4s
+; CHECK-GI-NEXT: addv s3, v3.4s
+; CHECK-GI-NEXT: mov v1.b[7], w8
+; CHECK-GI-NEXT: mov v2.b[3], w10
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: add w12, w13, w14
+; CHECK-GI-NEXT: fmov w13, s17
+; CHECK-GI-NEXT: fmov w10, s20
+; CHECK-GI-NEXT: ldr w14, [sp, #784]
+; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll v22.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: add w12, w12, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #792]
+; CHECK-GI-NEXT: mov v2.b[4], w9
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: fmov w12, s5
+; CHECK-GI-NEXT: addv s5, v4.4s
+; CHECK-GI-NEXT: ldr w9, [sp, #856]
+; CHECK-GI-NEXT: sshll v23.4s, v1.4h, #0
+; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: add w10, w10, w12
+; CHECK-GI-NEXT: fmov w12, s3
+; CHECK-GI-NEXT: fmov s3, w14
+; CHECK-GI-NEXT: ldr w14, [sp, #760]
+; CHECK-GI-NEXT: mul v22.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add w12, w10, w12
+; CHECK-GI-NEXT: ldr w10, [sp, #848]
+; CHECK-GI-NEXT: mov v3.b[1], w13
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: mov v2.b[5], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #920]
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #768]
+; CHECK-GI-NEXT: addv s1, v22.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: add w12, w12, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #800]
+; CHECK-GI-NEXT: mov v4.b[1], w9
+; CHECK-GI-NEXT: add w8, w11, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #912]
+; CHECK-GI-NEXT: mov v3.b[2], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #976]
+; CHECK-GI-NEXT: ldr w11, [sp, #984]
+; CHECK-GI-NEXT: fmov s5, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1048]
+; CHECK-GI-NEXT: mov v2.b[6], w10
+; CHECK-GI-NEXT: fmov s6, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #808]
+; CHECK-GI-NEXT: ldr w10, [sp, #832]
+; CHECK-GI-NEXT: mov v4.b[2], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #1040]
+; CHECK-GI-NEXT: ldr w9, [sp, #776]
+; CHECK-GI-NEXT: mov v5.b[1], w14
+; CHECK-GI-NEXT: mov v3.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1056]
+; CHECK-GI-NEXT: fmov s7, w15
+; CHECK-GI-NEXT: mov v6.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #872]
+; CHECK-GI-NEXT: ldr w14, [sp, #816]
+; CHECK-GI-NEXT: ldr w15, [sp, #992]
+; CHECK-GI-NEXT: mov v2.b[7], w9
+; CHECK-GI-NEXT: mov v4.b[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #824]
; CHECK-GI-NEXT: ldr w9, [sp, #840]
-; CHECK-GI-NEXT: ldr w11, [sp, #328]
-; CHECK-GI-NEXT: mov v4.b[14], w10
-; CHECK-GI-NEXT: mov v1.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #968]
-; CHECK-GI-NEXT: mov v5.b[14], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #1096]
-; CHECK-GI-NEXT: mov v3.b[15], w9
-; CHECK-GI-NEXT: mov v2.b[15], w11
-; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b
-; CHECK-GI-NEXT: mov v4.b[15], w8
-; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b
-; CHECK-GI-NEXT: mov v5.b[15], w10
-; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b
-; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b
-; CHECK-GI-NEXT: addv s0, v7.4s
-; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b
-; CHECK-GI-NEXT: addv s1, v16.4s
-; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b
-; CHECK-GI-NEXT: addv s2, v17.4s
-; CHECK-GI-NEXT: addv s4, v20.4s
-; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: mov v7.b[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #928]
+; CHECK-GI-NEXT: mov v3.b[4], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1064]
+; CHECK-GI-NEXT: mov v6.b[2], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #1000]
+; CHECK-GI-NEXT: mov v5.b[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #880]
+; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT: mov v7.b[2], w13
+; CHECK-GI-NEXT: mov v4.b[4], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1104]
+; CHECK-GI-NEXT: ldr w13, [sp, #936]
+; CHECK-GI-NEXT: mov v3.b[5], w11
+; CHECK-GI-NEXT: mov v6.b[3], w15
+; CHECK-GI-NEXT: fmov s16, w12
+; CHECK-GI-NEXT: ldr w12, [sp, #944]
+; CHECK-GI-NEXT: ldr w11, [sp, #1008]
+; CHECK-GI-NEXT: mov v5.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1112]
+; CHECK-GI-NEXT: ldr w15, [sp, #904]
+; CHECK-GI-NEXT: mov v7.b[3], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #888]
+; CHECK-GI-NEXT: mov v16.b[1], w13
+; CHECK-GI-NEXT: mov v3.b[6], w10
+; CHECK-GI-NEXT: ldr w13, [sp, #896]
+; CHECK-GI-NEXT: mov v4.b[5], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1072]
+; CHECK-GI-NEXT: mov v6.b[4], w11
+; CHECK-GI-NEXT: mov v5.b[4], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1120]
+; CHECK-GI-NEXT: ldr w11, [sp, #952]
+; CHECK-GI-NEXT: mov v7.b[4], w14
+; CHECK-GI-NEXT: ldr w10, [sp, #1016]
+; CHECK-GI-NEXT: ldr w14, [sp, #960]
+; CHECK-GI-NEXT: mov v16.b[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1080]
+; CHECK-GI-NEXT: mov v3.b[7], w9
+; CHECK-GI-NEXT: mov v4.b[6], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1128]
+; CHECK-GI-NEXT: ldr w9, [sp, #1088]
+; CHECK-GI-NEXT: mov v5.b[5], w11
+; CHECK-GI-NEXT: mov v6.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1024]
+; CHECK-GI-NEXT: mov v7.b[5], w12
+; CHECK-GI-NEXT: ldr w11, [sp, #968]
+; CHECK-GI-NEXT: ldr w12, [sp, #1032]
+; CHECK-GI-NEXT: mov v16.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1136]
+; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
+; CHECK-GI-NEXT: mov v4.b[7], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #1144]
+; CHECK-GI-NEXT: mov v5.b[6], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1168]
+; CHECK-GI-NEXT: mov v6.b[6], w10
+; CHECK-GI-NEXT: mov v7.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1232]
+; CHECK-GI-NEXT: ldr w10, [sp, #1176]
+; CHECK-GI-NEXT: mov v16.b[4], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1240]
+; CHECK-GI-NEXT: fmov s17, w14
+; CHECK-GI-NEXT: fmov s18, w9
+; CHECK-GI-NEXT: ldr w14, [sp, #1096]
+; CHECK-GI-NEXT: ldr w9, [sp, #1160]
+; CHECK-GI-NEXT: mov v5.b[7], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1184]
+; CHECK-GI-NEXT: mov v6.b[7], w12
+; CHECK-GI-NEXT: mov v17.b[1], w10
+; CHECK-GI-NEXT: ldr w12, [sp, #1248]
+; CHECK-GI-NEXT: mov v7.b[7], w14
+; CHECK-GI-NEXT: mov v18.b[1], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1296]
+; CHECK-GI-NEXT: mov v16.b[5], w15
+; CHECK-GI-NEXT: ldr w15, [sp, #1360]
+; CHECK-GI-NEXT: ldr w14, [sp, #1304]
+; CHECK-GI-NEXT: ldr w10, [sp, #1152]
+; CHECK-GI-NEXT: fmov s19, w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1424]
+; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT: mov v17.b[2], w11
+; CHECK-GI-NEXT: fmov s20, w15
+; CHECK-GI-NEXT: ldr w11, [sp, #1368]
+; CHECK-GI-NEXT: fmov s21, w13
+; CHECK-GI-NEXT: mov v18.b[2], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1432]
+; CHECK-GI-NEXT: mov v19.b[1], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1192]
+; CHECK-GI-NEXT: ldr w13, [sp, #1200]
+; CHECK-GI-NEXT: mov v20.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1312]
+; CHECK-GI-NEXT: mov v16.b[6], w10
+; CHECK-GI-NEXT: mov v21.b[1], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1256]
+; CHECK-GI-NEXT: mov v17.b[3], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1440]
+; CHECK-GI-NEXT: ldr w10, [sp, #1376]
+; CHECK-GI-NEXT: ushll v22.4s, v3.4h, #0
+; CHECK-GI-NEXT: mov v18.b[3], w12
+; CHECK-GI-NEXT: mov v19.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1208]
+; CHECK-GI-NEXT: mov v20.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1320]
+; CHECK-GI-NEXT: mov v16.b[7], w9
+; CHECK-GI-NEXT: mov v21.b[2], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1264]
+; CHECK-GI-NEXT: mov v17.b[4], w13
+; CHECK-GI-NEXT: ldr w9, [sp, #1384]
+; CHECK-GI-NEXT: ldr w13, [sp, #1448]
+; CHECK-GI-NEXT: ldr w12, [sp, #1216]
+; CHECK-GI-NEXT: mov v18.b[4], w14
+; CHECK-GI-NEXT: mov v19.b[3], w10
+; CHECK-GI-NEXT: ldr w14, [sp, #1272]
+; CHECK-GI-NEXT: mov v20.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1328]
+; CHECK-GI-NEXT: ldr w10, [sp, #1224]
+; CHECK-GI-NEXT: mov v17.b[5], w11
+; CHECK-GI-NEXT: mov v21.b[3], w13
+; CHECK-GI-NEXT: ldr w13, [sp, #1280]
+; CHECK-GI-NEXT: ldr w11, [sp, #1392]
+; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT: mov v18.b[5], w14
+; CHECK-GI-NEXT: mov v19.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1336]
+; CHECK-GI-NEXT: ldr w14, [sp, #1456]
+; CHECK-GI-NEXT: mov v20.b[4], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1288]
+; CHECK-GI-NEXT: mov v17.b[6], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1344]
+; CHECK-GI-NEXT: sshll v27.4s, v16.4h, #0
+; CHECK-GI-NEXT: mov v21.b[4], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1400]
+; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT: mov v18.b[6], w13
+; CHECK-GI-NEXT: mov v19.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1464]
+; CHECK-GI-NEXT: mov v20.b[5], w14
+; CHECK-GI-NEXT: ldr w14, [sp, #1352]
+; CHECK-GI-NEXT: ldr w13, [sp, #1416]
+; CHECK-GI-NEXT: mov v17.b[7], w10
+; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0
+; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
+; CHECK-GI-NEXT: mov v21.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1408]
+; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT: mov v18.b[7], w11
+; CHECK-GI-NEXT: mov v19.b[6], w12
+; CHECK-GI-NEXT: ldr w12, [sp, #1472]
+; CHECK-GI-NEXT: mov v20.b[6], w9
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: addv s3, v18.4s
-; CHECK-GI-NEXT: addv s5, v19.4s
-; CHECK-GI-NEXT: fmov w10, s2
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s4
-; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: ushll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
+; CHECK-GI-NEXT: ldr w15, [sp, #1480]
+; CHECK-GI-NEXT: mov v21.b[6], w12
+; CHECK-GI-NEXT: ushll v24.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
+; CHECK-GI-NEXT: sshll v18.8h, v18.8b, #0
+; CHECK-GI-NEXT: mov v19.b[7], w14
+; CHECK-GI-NEXT: mul v1.4s, v1.4s, v27.4s
+; CHECK-GI-NEXT: sshll v28.4s, v17.4h, #0
+; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
+; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s
+; CHECK-GI-NEXT: mov v20.b[7], w13
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: ushll v25.4s, v6.4h, #0
+; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0
+; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
+; CHECK-GI-NEXT: mov v21.b[7], w15
+; CHECK-GI-NEXT: sshll v19.8h, v19.8b, #0
+; CHECK-GI-NEXT: mul v16.4s, v22.4s, v28.4s
+; CHECK-GI-NEXT: mul v3.4s, v3.4s, v17.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: addv s2, v2.4s
+; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0
+; CHECK-GI-NEXT: mul v17.4s, v23.4s, v29.4s
+; CHECK-GI-NEXT: sshll v20.8h, v20.8b, #0
+; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s
+; CHECK-GI-NEXT: sshll v30.4s, v19.4h, #0
+; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0
+; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
+; CHECK-GI-NEXT: addv s16, v16.4s
+; CHECK-GI-NEXT: addv s3, v3.4s
+; CHECK-GI-NEXT: fmov w10, s1
+; CHECK-GI-NEXT: fmov w11, s2
+; CHECK-GI-NEXT: sshll v31.4s, v20.4h, #0
+; CHECK-GI-NEXT: sshll v21.8h, v21.8b, #0
+; CHECK-GI-NEXT: addv s17, v17.4s
+; CHECK-GI-NEXT: mul v18.4s, v24.4s, v30.4s
+; CHECK-GI-NEXT: mul v5.4s, v5.4s, v19.4s
+; CHECK-GI-NEXT: addv s4, v4.4s
+; CHECK-GI-NEXT: fmov w12, s3
+; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: fmov w11, s16
+; CHECK-GI-NEXT: mul v19.4s, v25.4s, v31.4s
+; CHECK-GI-NEXT: fmov w13, s17
+; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: addv s18, v18.4s
+; CHECK-GI-NEXT: addv s5, v5.4s
+; CHECK-GI-NEXT: sshll v27.4s, v21.4h, #0
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: sshll2 v21.4s, v21.8h, #0
+; CHECK-GI-NEXT: mul v6.4s, v6.4s, v20.4s
+; CHECK-GI-NEXT: add w11, w12, w13
+; CHECK-GI-NEXT: fmov w12, s4
+; CHECK-GI-NEXT: addv s19, v19.4s
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: mul v20.4s, v26.4s, v27.4s
+; CHECK-GI-NEXT: fmov w14, s0
+; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: fmov w12, s18
+; CHECK-GI-NEXT: addv s6, v6.4s
+; CHECK-GI-NEXT: add w10, w10, w11
+; CHECK-GI-NEXT: add w9, w9, w14
+; CHECK-GI-NEXT: addv s1, v20.4s
; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w12, w12, w13
+; CHECK-GI-NEXT: fmov w13, s19
+; CHECK-GI-NEXT: addv s2, v7.4s
+; CHECK-GI-NEXT: add w11, w12, w13
+; CHECK-GI-NEXT: fmov w12, s6
+; CHECK-GI-NEXT: fmov w13, s2
+; CHECK-GI-NEXT: add w11, w11, w12
+; CHECK-GI-NEXT: fmov w12, s1
; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: fmov w11, s5
+; CHECK-GI-NEXT: add w11, w12, w13
; CHECK-GI-NEXT: add w9, w10, w11
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
- %az = sext <48 x i8> %a to <48 x i32>
- %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
- %cz = sext <48 x i8> %c to <48 x i32>
- %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
+ %az = zext <48 x i8> %a to <48 x i32>
+ %bz = sext <48 x i8> %b to <48 x i32>
+ %m1 = mul nuw nsw <48 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
+ %cz = zext <48 x i8> %c to <48 x i32>
+ %dz = sext <48 x i8> %d to <48 x i32>
+ %m2 = mul nuw nsw <48 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
@@ -4527,3 +7593,385 @@ entry:
%x = add i32 %r1, %r2
ret i32 %x
}
+
+define i32 @test_usdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_usdot_v64i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: ldp q1, q2, [x0, #32]
+; CHECK-SD-NEXT: ldp q6, q7, [x1, #32]
+; CHECK-SD-NEXT: ldp q16, q17, [x0]
+; CHECK-SD-NEXT: ldp q18, q19, [x1]
+; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v7.16b
+; CHECK-SD-NEXT: usdot v5.4s, v1.16b, v6.16b
+; CHECK-SD-NEXT: usdot v4.4s, v17.16b, v19.16b
+; CHECK-SD-NEXT: usdot v3.4s, v16.16b, v18.16b
+; CHECK-SD-NEXT: add v0.4s, v4.4s, v0.4s
+; CHECK-SD-NEXT: add v1.4s, v3.4s, v5.4s
+; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v64i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT: .cfi_offset b8, -8
+; CHECK-GI-NEXT: .cfi_offset b9, -16
+; CHECK-GI-NEXT: .cfi_offset b10, -24
+; CHECK-GI-NEXT: .cfi_offset b11, -32
+; CHECK-GI-NEXT: .cfi_offset b12, -40
+; CHECK-GI-NEXT: .cfi_offset b13, -48
+; CHECK-GI-NEXT: .cfi_offset b14, -56
+; CHECK-GI-NEXT: .cfi_offset b15, -64
+; CHECK-GI-NEXT: ldp q0, q1, [x1]
+; CHECK-GI-NEXT: ldp q21, q17, [x0]
+; CHECK-GI-NEXT: ldp q3, q19, [x1, #32]
+; CHECK-GI-NEXT: ldp q18, q4, [x0, #32]
+; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v5.8h, v0.16b, #0
+; CHECK-GI-NEXT: sshll v7.8h, v1.8b, #0
+; CHECK-GI-NEXT: sshll2 v22.8h, v1.16b, #0
+; CHECK-GI-NEXT: sshll v23.8h, v3.8b, #0
+; CHECK-GI-NEXT: sshll2 v24.8h, v3.16b, #0
+; CHECK-GI-NEXT: sshll v25.8h, v19.8b, #0
+; CHECK-GI-NEXT: sshll2 v26.8h, v19.16b, #0
+; CHECK-GI-NEXT: ushll v27.8h, v21.8b, #0
+; CHECK-GI-NEXT: ushll2 v28.8h, v21.16b, #0
+; CHECK-GI-NEXT: ushll v30.8h, v17.8b, #0
+; CHECK-GI-NEXT: ushll2 v17.8h, v17.16b, #0
+; CHECK-GI-NEXT: ushll v8.8h, v18.8b, #0
+; CHECK-GI-NEXT: ushll2 v18.8h, v18.16b, #0
+; CHECK-GI-NEXT: ushll v9.8h, v4.8b, #0
+; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-GI-NEXT: sshll v0.4s, v2.4h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v2.8h, #0
+; CHECK-GI-NEXT: sshll v1.4s, v5.4h, #0
+; CHECK-GI-NEXT: sshll2 v16.4s, v5.8h, #0
+; CHECK-GI-NEXT: sshll v2.4s, v7.4h, #0
+; CHECK-GI-NEXT: sshll2 v20.4s, v7.8h, #0
+; CHECK-GI-NEXT: sshll v3.4s, v22.4h, #0
+; CHECK-GI-NEXT: sshll2 v22.4s, v22.8h, #0
+; CHECK-GI-NEXT: sshll v5.4s, v23.4h, #0
+; CHECK-GI-NEXT: sshll2 v23.4s, v23.8h, #0
+; CHECK-GI-NEXT: sshll v7.4s, v24.4h, #0
+; CHECK-GI-NEXT: sshll2 v24.4s, v24.8h, #0
+; CHECK-GI-NEXT: sshll v19.4s, v25.4h, #0
+; CHECK-GI-NEXT: sshll2 v25.4s, v25.8h, #0
+; CHECK-GI-NEXT: sshll v21.4s, v26.4h, #0
+; CHECK-GI-NEXT: sshll2 v26.4s, v26.8h, #0
+; CHECK-GI-NEXT: ushll v29.4s, v27.4h, #0
+; CHECK-GI-NEXT: ushll2 v27.4s, v27.8h, #0
+; CHECK-GI-NEXT: ushll v31.4s, v28.4h, #0
+; CHECK-GI-NEXT: ushll2 v28.4s, v28.8h, #0
+; CHECK-GI-NEXT: ushll v10.4s, v30.4h, #0
+; CHECK-GI-NEXT: ushll2 v30.4s, v30.8h, #0
+; CHECK-GI-NEXT: ushll v11.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll2 v17.4s, v17.8h, #0
+; CHECK-GI-NEXT: ushll2 v12.4s, v8.8h, #0
+; CHECK-GI-NEXT: ushll2 v13.4s, v18.8h, #0
+; CHECK-GI-NEXT: ushll2 v14.4s, v9.8h, #0
+; CHECK-GI-NEXT: ushll2 v15.4s, v4.8h, #0
+; CHECK-GI-NEXT: mul v6.4s, v6.4s, v27.4s
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v28.4s
+; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s
+; CHECK-GI-NEXT: mul v17.4s, v22.4s, v17.4s
+; CHECK-GI-NEXT: ushll v8.4s, v8.4h, #0
+; CHECK-GI-NEXT: mul v22.4s, v23.4s, v12.4s
+; CHECK-GI-NEXT: mul v23.4s, v24.4s, v13.4s
+; CHECK-GI-NEXT: mul v24.4s, v25.4s, v14.4s
+; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mul v25.4s, v26.4s, v15.4s
+; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT: ushll v26.4s, v9.4h, #0
+; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: mla v6.4s, v0.4s, v29.4s
+; CHECK-GI-NEXT: mla v16.4s, v1.4s, v31.4s
+; CHECK-GI-NEXT: mla v20.4s, v2.4s, v10.4s
+; CHECK-GI-NEXT: mla v17.4s, v3.4s, v11.4s
+; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mla v22.4s, v5.4s, v8.4s
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mla v23.4s, v7.4s, v18.4s
+; CHECK-GI-NEXT: mla v24.4s, v19.4s, v26.4s
+; CHECK-GI-NEXT: mla v25.4s, v21.4s, v4.4s
+; CHECK-GI-NEXT: add v0.4s, v6.4s, v16.4s
+; CHECK-GI-NEXT: add v1.4s, v20.4s, v17.4s
+; CHECK-GI-NEXT: add v2.4s, v22.4s, v23.4s
+; CHECK-GI-NEXT: add v3.4s, v24.4s, v25.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <64 x i8>, ptr %a
+ %1 = zext <64 x i8> %0 to <64 x i32>
+ %2 = load <64 x i8>, ptr %b
+ %3 = sext <64 x i8> %2 to <64 x i32>
+ %4 = mul nsw <64 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_usdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
+; CHECK-SD-LABEL: test_usdot_v64i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v21.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v22.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v23.2d, #0000000000000000
+; CHECK-SD-NEXT: ldp q16, q17, [sp, #64]
+; CHECK-SD-NEXT: movi v24.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v25.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v26.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v27.2d, #0000000000000000
+; CHECK-SD-NEXT: ldp q19, q20, [sp, #96]
+; CHECK-SD-NEXT: usdot v18.4s, v3.16b, v7.16b
+; CHECK-SD-NEXT: ldp q3, q7, [sp, #32]
+; CHECK-SD-NEXT: usdot v21.4s, v1.16b, v5.16b
+; CHECK-SD-NEXT: ldp q1, q5, [sp]
+; CHECK-SD-NEXT: usdot v22.4s, v2.16b, v6.16b
+; CHECK-SD-NEXT: usdot v23.4s, v0.16b, v4.16b
+; CHECK-SD-NEXT: usdot v24.4s, v7.16b, v20.16b
+; CHECK-SD-NEXT: usdot v27.4s, v3.16b, v19.16b
+; CHECK-SD-NEXT: usdot v26.4s, v5.16b, v17.16b
+; CHECK-SD-NEXT: usdot v25.4s, v1.16b, v16.16b
+; CHECK-SD-NEXT: add v0.4s, v21.4s, v18.4s
+; CHECK-SD-NEXT: add v1.4s, v23.4s, v22.4s
+; CHECK-SD-NEXT: add v2.4s, v26.4s, v24.4s
+; CHECK-SD-NEXT: add v3.4s, v25.4s, v27.4s
+; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: add v1.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_usdot_v64i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: sub sp, sp, #304
+; CHECK-GI-NEXT: stp d15, d14, [sp, #224] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d13, d12, [sp, #240] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d11, d10, [sp, #256] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp d9, d8, [sp, #272] // 16-byte Folded Spill
+; CHECK-GI-NEXT: str x29, [sp, #288] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 304
+; CHECK-GI-NEXT: .cfi_offset w29, -16
+; CHECK-GI-NEXT: .cfi_offset b8, -24
+; CHECK-GI-NEXT: .cfi_offset b9, -32
+; CHECK-GI-NEXT: .cfi_offset b10, -40
+; CHECK-GI-NEXT: .cfi_offset b11, -48
+; CHECK-GI-NEXT: .cfi_offset b12, -56
+; CHECK-GI-NEXT: .cfi_offset b13, -64
+; CHECK-GI-NEXT: .cfi_offset b14, -72
+; CHECK-GI-NEXT: .cfi_offset b15, -80
+; CHECK-GI-NEXT: ushll v17.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
+; CHECK-GI-NEXT: ldr x29, [sp, #288] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v20.16b, v3.16b
+; CHECK-GI-NEXT: ushll v16.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v18.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v26.8h, v2.8b, #0
+; CHECK-GI-NEXT: ldp q27, q28, [sp, #304]
+; CHECK-GI-NEXT: ushll2 v29.8h, v2.16b, #0
+; CHECK-GI-NEXT: ushll v2.4s, v17.4h, #0
+; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT: sshll v8.8h, v4.8b, #0
+; CHECK-GI-NEXT: ldp q23, q21, [sp, #368]
+; CHECK-GI-NEXT: sshll2 v9.8h, v4.16b, #0
+; CHECK-GI-NEXT: sshll2 v11.8h, v5.16b, #0
+; CHECK-GI-NEXT: mov v25.16b, v7.16b
+; CHECK-GI-NEXT: ushll2 v19.4s, v17.8h, #0
+; CHECK-GI-NEXT: stp q1, q2, [sp, #192] // 32-byte Folded Spill
+; CHECK-GI-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll2 v17.4s, v18.8h, #0
+; CHECK-GI-NEXT: ldp q24, q22, [sp, #336]
+; CHECK-GI-NEXT: sshll v10.8h, v5.8b, #0
+; CHECK-GI-NEXT: sshll v12.8h, v6.8b, #0
+; CHECK-GI-NEXT: sshll2 v13.8h, v6.16b, #0
+; CHECK-GI-NEXT: mov v2.16b, v20.16b
+; CHECK-GI-NEXT: sshll2 v0.4s, v8.8h, #0
+; CHECK-GI-NEXT: sshll2 v4.4s, v9.8h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v11.8h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v16.8h, #0
+; CHECK-GI-NEXT: ushll2 v31.4s, v29.8h, #0
+; CHECK-GI-NEXT: sshll2 v5.4s, v10.8h, #0
+; CHECK-GI-NEXT: sshll2 v1.4s, v13.8h, #0
+; CHECK-GI-NEXT: ushll2 v30.4s, v26.8h, #0
+; CHECK-GI-NEXT: ushll v14.8h, v2.8b, #0
+; CHECK-GI-NEXT: mul v20.4s, v19.4s, v0.4s
+; CHECK-GI-NEXT: mul v19.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT: sshll v0.8h, v25.8b, #0
+; CHECK-GI-NEXT: mul v4.4s, v17.4s, v6.4s
+; CHECK-GI-NEXT: sshll2 v15.4s, v12.8h, #0
+; CHECK-GI-NEXT: ldp q17, q3, [sp, #400]
+; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s
+; CHECK-GI-NEXT: mul v7.4s, v31.4s, v1.4s
+; CHECK-GI-NEXT: ushll2 v31.8h, v2.16b, #0
+; CHECK-GI-NEXT: sshll2 v25.8h, v25.16b, #0
+; CHECK-GI-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v14.4h, #0
+; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: str q3, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT: ushll2 v3.4s, v14.8h, #0
+; CHECK-GI-NEXT: mul v6.4s, v30.4s, v15.4s
+; CHECK-GI-NEXT: str q31, [sp, #160] // 16-byte Folded Spill
+; CHECK-GI-NEXT: ushll v30.4s, v26.4h, #0
+; CHECK-GI-NEXT: sshll v26.4s, v8.4h, #0
+; CHECK-GI-NEXT: ushll v14.8h, v27.8b, #0
+; CHECK-GI-NEXT: ushll v15.4s, v29.4h, #0
+; CHECK-GI-NEXT: sshll v29.4s, v9.4h, #0
+; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s
+; CHECK-GI-NEXT: ushll2 v3.4s, v31.8h, #0
+; CHECK-GI-NEXT: ushll v31.8h, v28.8b, #0
+; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0
+; CHECK-GI-NEXT: sshll v8.4s, v10.4h, #0
+; CHECK-GI-NEXT: sshll v9.4s, v11.4h, #0
+; CHECK-GI-NEXT: sshll v10.4s, v12.4h, #0
+; CHECK-GI-NEXT: sshll v11.4s, v13.4h, #0
+; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT: stp q3, q25, [sp, #112] // 32-byte Folded Spill
+; CHECK-GI-NEXT: ldr q3, [sp, #208] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll2 v28.8h, v28.16b, #0
+; CHECK-GI-NEXT: mla v1.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: ushll2 v0.4s, v31.8h, #0
+; CHECK-GI-NEXT: mla v5.4s, v16.4s, v8.4s
+; CHECK-GI-NEXT: mla v20.4s, v3.4s, v26.4s
+; CHECK-GI-NEXT: sshll2 v3.4s, v25.8h, #0
+; CHECK-GI-NEXT: mla v6.4s, v30.4s, v10.4s
+; CHECK-GI-NEXT: mla v7.4s, v15.4s, v11.4s
+; CHECK-GI-NEXT: sshll v25.8h, v23.8b, #0
+; CHECK-GI-NEXT: mla v4.4s, v18.4s, v9.4s
+; CHECK-GI-NEXT: ushll v30.8h, v22.8b, #0
+; CHECK-GI-NEXT: ushll2 v26.8h, v22.16b, #0
+; CHECK-GI-NEXT: sshll v22.8h, v21.8b, #0
+; CHECK-GI-NEXT: str q3, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: ldr q3, [sp, #192] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll2 v8.8h, v27.16b, #0
+; CHECK-GI-NEXT: str q1, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: ldr q9, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll2 v1.4s, v14.8h, #0
+; CHECK-GI-NEXT: stp q7, q6, [sp, #64] // 32-byte Folded Spill
+; CHECK-GI-NEXT: mla v19.4s, v3.4s, v29.4s
+; CHECK-GI-NEXT: sshll2 v7.4s, v25.8h, #0
+; CHECK-GI-NEXT: str q5, [sp, #176] // 16-byte Folded Spill
+; CHECK-GI-NEXT: ushll v29.8h, v24.8b, #0
+; CHECK-GI-NEXT: ushll2 v27.8h, v24.16b, #0
+; CHECK-GI-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill
+; CHECK-GI-NEXT: ldp q0, q16, [sp, #96] // 32-byte Folded Reload
+; CHECK-GI-NEXT: str q4, [sp, #144] // 16-byte Folded Spill
+; CHECK-GI-NEXT: sshll2 v24.8h, v23.16b, #0
+; CHECK-GI-NEXT: ushll2 v18.4s, v26.8h, #0
+; CHECK-GI-NEXT: stp q19, q20, [sp, #192] // 32-byte Folded Spill
+; CHECK-GI-NEXT: sshll2 v20.8h, v21.16b, #0
+; CHECK-GI-NEXT: sshll v21.8h, v17.8b, #0
+; CHECK-GI-NEXT: sshll2 v19.8h, v17.16b, #0
+; CHECK-GI-NEXT: sshll2 v17.8h, v0.16b, #0
+; CHECK-GI-NEXT: mul v16.4s, v16.4s, v9.4s
+; CHECK-GI-NEXT: ldr q9, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sshll v23.8h, v0.8b, #0
+; CHECK-GI-NEXT: sshll2 v2.4s, v22.8h, #0
+; CHECK-GI-NEXT: ushll2 v12.4s, v27.8h, #0
+; CHECK-GI-NEXT: ushll v26.4s, v26.4h, #0
+; CHECK-GI-NEXT: ushll2 v10.4s, v28.8h, #0
+; CHECK-GI-NEXT: sshll2 v0.4s, v17.8h, #0
+; CHECK-GI-NEXT: mul v7.4s, v9.4s, v7.4s
+; CHECK-GI-NEXT: ldr q9, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sshll2 v5.4s, v19.8h, #0
+; CHECK-GI-NEXT: sshll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT: sshll2 v3.4s, v20.8h, #0
+; CHECK-GI-NEXT: mul v2.4s, v9.4s, v2.4s
+; CHECK-GI-NEXT: ldr q9, [sp, #128] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll2 v15.4s, v8.8h, #0
+; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s
+; CHECK-GI-NEXT: ldr q18, [sp, #160] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll2 v11.4s, v29.8h, #0
+; CHECK-GI-NEXT: sshll v9.4s, v9.4h, #0
+; CHECK-GI-NEXT: ushll2 v13.4s, v30.8h, #0
+; CHECK-GI-NEXT: sshll2 v1.4s, v24.8h, #0
+; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0
+; CHECK-GI-NEXT: sshll2 v4.4s, v21.8h, #0
+; CHECK-GI-NEXT: sshll2 v6.4s, v23.8h, #0
+; CHECK-GI-NEXT: mul v5.4s, v12.4s, v5.4s
+; CHECK-GI-NEXT: ushll v27.4s, v27.4h, #0
+; CHECK-GI-NEXT: sshll v19.4s, v19.4h, #0
+; CHECK-GI-NEXT: mla v0.4s, v26.4s, v17.4s
+; CHECK-GI-NEXT: mul v3.4s, v10.4s, v3.4s
+; CHECK-GI-NEXT: mul v1.4s, v15.4s, v1.4s
+; CHECK-GI-NEXT: mla v16.4s, v18.4s, v9.4s
+; CHECK-GI-NEXT: ldp q18, q17, [sp, #192] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mul v4.4s, v11.4s, v4.4s
+; CHECK-GI-NEXT: mul v6.4s, v13.4s, v6.4s
+; CHECK-GI-NEXT: ushll v28.4s, v28.4h, #0
+; CHECK-GI-NEXT: ldp d13, d12, [sp, #240] // 16-byte Folded Reload
+; CHECK-GI-NEXT: sshll v20.4s, v20.4h, #0
+; CHECK-GI-NEXT: ushll v10.4s, v14.4h, #0
+; CHECK-GI-NEXT: ldp d15, d14, [sp, #224] // 16-byte Folded Reload
+; CHECK-GI-NEXT: ushll v8.4s, v8.4h, #0
+; CHECK-GI-NEXT: ushll v31.4s, v31.4h, #0
+; CHECK-GI-NEXT: ushll v29.4s, v29.4h, #0
+; CHECK-GI-NEXT: ushll v30.4s, v30.4h, #0
+; CHECK-GI-NEXT: sshll v25.4s, v25.4h, #0
+; CHECK-GI-NEXT: sshll v24.4s, v24.4h, #0
+; CHECK-GI-NEXT: sshll v22.4s, v22.4h, #0
+; CHECK-GI-NEXT: sshll v21.4s, v21.4h, #0
+; CHECK-GI-NEXT: sshll v23.4s, v23.4h, #0
+; CHECK-GI-NEXT: mla v5.4s, v27.4s, v19.4s
+; CHECK-GI-NEXT: ldr q19, [sp, #144] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v17.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: ldr q18, [sp, #176] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mla v3.4s, v28.4s, v20.4s
+; CHECK-GI-NEXT: mla v7.4s, v10.4s, v25.4s
+; CHECK-GI-NEXT: ldp d11, d10, [sp, #256] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mla v1.4s, v8.4s, v24.4s
+; CHECK-GI-NEXT: ldp d9, d8, [sp, #272] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v18.4s, v18.4s, v19.4s
+; CHECK-GI-NEXT: ldp q20, q19, [sp, #64] // 32-byte Folded Reload
+; CHECK-GI-NEXT: mla v2.4s, v31.4s, v22.4s
+; CHECK-GI-NEXT: mla v4.4s, v29.4s, v21.4s
+; CHECK-GI-NEXT: mla v6.4s, v30.4s, v23.4s
+; CHECK-GI-NEXT: add v1.4s, v7.4s, v1.4s
+; CHECK-GI-NEXT: add v19.4s, v19.4s, v20.4s
+; CHECK-GI-NEXT: ldr q20, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: add v16.4s, v20.4s, v16.4s
+; CHECK-GI-NEXT: add v3.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: add v0.4s, v6.4s, v0.4s
+; CHECK-GI-NEXT: add v4.4s, v17.4s, v18.4s
+; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: add v5.4s, v19.4s, v16.4s
+; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: addv s1, v2.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: fmov w9, s0
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: add sp, sp, #304
+; CHECK-GI-NEXT: ret
+entry:
+ %az = zext <64 x i8> %a to <64 x i32>
+ %bz = sext <64 x i8> %b to <64 x i32>
+ %m1 = mul nuw nsw <64 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)
+ %cz = zext <64 x i8> %c to <64 x i32>
+ %dz = sext <64 x i8> %d to <64 x i32>
+ %m2 = mul nuw nsw <64 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
>From 31e138df235e5c42e67c4195e7ee8d55c0496a01 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 18 Dec 2024 13:40:45 +0000
Subject: [PATCH 2/2] Remove some tests
---
llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 5298 ++++++-------------
1 file changed, 1475 insertions(+), 3823 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 05ac2956da00c7..748555d7bdfa15 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -7,22 +7,16 @@
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8_double
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
@@ -542,65 +536,6 @@ entry:
ret i32 %x
}
-define i32 @test_usdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_usdot_v5i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h
-; CHECK-NEXT: mov v3.s[0], v2.s[0]
-; CHECK-NEXT: smlal v3.4s, v1.4h, v0.4h
-; CHECK-NEXT: addv s0, v3.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <5 x i8>, ptr %a
- %1 = zext <5 x i8> %0 to <5 x i32>
- %2 = load <5 x i8>, ptr %b
- %3 = sext <5 x i8> %2 to <5 x i32>
- %4 = mul nsw <5 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_usdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
-; CHECK-LABEL: test_usdot_v5i8_double:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ushll v2.8h, v2.8b, #0
-; CHECK-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: movi v6.2d, #0000000000000000
-; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h
-; CHECK-NEXT: smull2 v7.4s, v2.8h, v3.8h
-; CHECK-NEXT: mov v6.s[0], v4.s[0]
-; CHECK-NEXT: mov v5.s[0], v7.s[0]
-; CHECK-NEXT: smlal v6.4s, v0.4h, v1.4h
-; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h
-; CHECK-NEXT: add v0.4s, v6.4s, v5.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
-entry:
- %az = zext <5 x i8> %a to <5 x i32>
- %bz = sext <5 x i8> %b to <5 x i32>
- %m1 = mul nuw nsw <5 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)
- %cz = zext <5 x i8> %c to <5 x i32>
- %dz = sext <5 x i8> %d to <5 x i32>
- %m2 = mul nuw nsw <5 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)
- %x = add i32 %r1, %r2
- ret i32 %x
-}
-
-
define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {
; CHECK-LABEL: test_udot_v8i8:
; CHECK: // %bb.0: // %entry
@@ -2245,671 +2180,100 @@ entry:
ret i32 %x
}
-define i32 @test_usdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_usdot_v24i8:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q2, [x0]
-; CHECK-SD-NEXT: ldr q3, [x1]
-; CHECK-SD-NEXT: ldr d4, [x1, #16]
-; CHECK-SD-NEXT: ldr d5, [x0, #16]
-; CHECK-SD-NEXT: usdot v1.2s, v5.8b, v4.8b
-; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v3.16b
-; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s
-; CHECK-SD-NEXT: addv s0, v0.4s
-; CHECK-SD-NEXT: fmov w8, s1
-; CHECK-SD-NEXT: fmov w9, s0
-; CHECK-SD-NEXT: add w8, w9, w8
-; CHECK-SD-NEXT: add w0, w8, w2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_usdot_v24i8:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldr q0, [x0]
-; CHECK-GI-NEXT: ldr d1, [x0, #16]
-; CHECK-GI-NEXT: ldr q2, [x1]
-; CHECK-GI-NEXT: ldr d3, [x1, #16]
-; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
-; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT: sshll v5.8h, v2.8b, #0
-; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-GI-NEXT: ushll v6.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: ushll v7.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: sshll v16.4s, v5.4h, #0
-; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v17.4s, v1.4h, #0
-; CHECK-GI-NEXT: sshll2 v18.4s, v2.8h, #0
-; CHECK-GI-NEXT: sshll v19.4s, v3.4h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0
-; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT: mul v6.4s, v16.4s, v6.4s
-; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s
-; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s
-; CHECK-GI-NEXT: mul v5.4s, v19.4s, v17.4s
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v7.4s
-; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s
-; CHECK-GI-NEXT: addv s3, v6.4s
-; CHECK-GI-NEXT: addv s4, v4.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: addv s5, v5.4s
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: fmov w8, s3
-; CHECK-GI-NEXT: fmov w9, s4
-; CHECK-GI-NEXT: fmov w10, s0
-; CHECK-GI-NEXT: fmov w11, s5
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: fmov w11, s1
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w9, w10, w11
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w0, w8, w2
-; CHECK-GI-NEXT: ret
+define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_udot_v25i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q3, q0, [x1]
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b
+; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b
+; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b
+; CHECK-NEXT: ushll v3.4s, v4.4h, #0
+; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
+; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h
+; CHECK-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
entry:
- %0 = load <24 x i8>, ptr %a
- %1 = zext <24 x i8> %0 to <24 x i32>
- %2 = load <24 x i8>, ptr %b
- %3 = sext <24 x i8> %2 to <24 x i32>
- %4 = mul nsw <24 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)
+ %0 = load <25 x i8>, ptr %a
+ %1 = zext <25 x i8> %0 to <25 x i32>
+ %2 = load <25 x i8>, ptr %b
+ %3 = zext <25 x i8> %2 to <25 x i32>
+ %4 = mul nuw nsw <25 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
+ %op.extra = add i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-LABEL: test_udot_v25i8_nomla:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll v4.8h, v2.8b, #0
+; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
+; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h
+; CHECK-NEXT: mov v0.s[0], v3.s[0]
+; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
+; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a1
+ %1 = zext <25 x i8> %0 to <25 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
+ ret i32 %2
+}
+define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-LABEL: test_sdot_v25i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q3, q0, [x1]
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b
+; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b
+; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b
+; CHECK-NEXT: sshll v3.4s, v4.4h, #0
+; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h
+; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h
+; CHECK-NEXT: mov v5.s[0], v3.s[0]
+; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w2
+; CHECK-NEXT: ret
+entry:
+ %0 = load <25 x i8>, ptr %a
+ %1 = sext <25 x i8> %0 to <25 x i32>
+ %2 = load <25 x i8>, ptr %b
+ %3 = sext <25 x i8> %2 to <25 x i32>
+ %4 = mul nsw <25 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
%op.extra = add nsw i32 %5, %sum
ret i32 %op.extra
}
-define i32 @test_usdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
-; CHECK-SD-LABEL: test_usdot_v24i8_double:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: .cfi_offset w29, -16
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: ldr b1, [sp, #144]
-; CHECK-SD-NEXT: add x10, sp, #152
-; CHECK-SD-NEXT: add x9, sp, #160
-; CHECK-SD-NEXT: add x8, sp, #168
-; CHECK-SD-NEXT: ldr b2, [sp, #272]
-; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10]
-; CHECK-SD-NEXT: add x11, sp, #280
-; CHECK-SD-NEXT: ldr b3, [sp, #80]
-; CHECK-SD-NEXT: mov v0.b[1], w1
-; CHECK-SD-NEXT: ldr b4, [sp, #528]
-; CHECK-SD-NEXT: add x10, sp, #88
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #536
-; CHECK-SD-NEXT: ldr b5, [sp, #336]
-; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #344
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
-; CHECK-SD-NEXT: add x11, sp, #176
-; CHECK-SD-NEXT: ldr b6, [sp, #656]
-; CHECK-SD-NEXT: mov v0.b[2], w2
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
-; CHECK-SD-NEXT: ldr b7, [sp, #464]
-; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8]
-; CHECK-SD-NEXT: add x12, sp, #664
-; CHECK-SD-NEXT: add x9, sp, #472
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
-; CHECK-SD-NEXT: add x8, sp, #96
-; CHECK-SD-NEXT: add x10, sp, #184
-; CHECK-SD-NEXT: add x12, sp, #288
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
-; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
-; CHECK-SD-NEXT: mov v0.b[3], w3
-; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11]
-; CHECK-SD-NEXT: add x8, sp, #352
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
-; CHECK-SD-NEXT: add x13, sp, #544
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #672
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x13]
-; CHECK-SD-NEXT: add x9, sp, #192
-; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #480
-; CHECK-SD-NEXT: mov v0.b[4], w4
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #296
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #552
-; CHECK-SD-NEXT: add x12, sp, #200
-; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #360
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #104
-; CHECK-SD-NEXT: add x9, sp, #560
-; CHECK-SD-NEXT: mov v0.b[5], w5
-; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #368
-; CHECK-SD-NEXT: ld1 { v1.b }[7], [x12]
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9]
-; CHECK-SD-NEXT: add x13, sp, #208
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
-; CHECK-SD-NEXT: add x12, sp, #304
-; CHECK-SD-NEXT: add x8, sp, #568
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12]
-; CHECK-SD-NEXT: add x12, sp, #16
-; CHECK-SD-NEXT: add x17, sp, #376
-; CHECK-SD-NEXT: mov v0.b[6], w6
-; CHECK-SD-NEXT: ld1 { v1.b }[8], [x13]
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8]
-; CHECK-SD-NEXT: add x14, sp, #216
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x17]
-; CHECK-SD-NEXT: add x13, sp, #576
-; CHECK-SD-NEXT: add x11, sp, #224
-; CHECK-SD-NEXT: add x10, sp, #232
-; CHECK-SD-NEXT: add x15, sp, #240
-; CHECK-SD-NEXT: ld1 { v1.b }[9], [x14]
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
-; CHECK-SD-NEXT: add x13, sp, #384
-; CHECK-SD-NEXT: mov v0.b[7], w7
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x13]
-; CHECK-SD-NEXT: add x13, sp, #112
-; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13]
-; CHECK-SD-NEXT: add x13, sp, #32
-; CHECK-SD-NEXT: add x14, sp, #584
-; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x14]
-; CHECK-SD-NEXT: add x11, sp, #312
-; CHECK-SD-NEXT: add x14, sp, #40
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #592
-; CHECK-SD-NEXT: ld1 { v0.b }[8], [x12]
-; CHECK-SD-NEXT: add x12, sp, #24
-; CHECK-SD-NEXT: add x16, sp, #248
-; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10]
-; CHECK-SD-NEXT: ld1 { v4.b }[8], [x11]
-; CHECK-SD-NEXT: add x11, sp, #400
-; CHECK-SD-NEXT: add x9, sp, #256
-; CHECK-SD-NEXT: add x8, sp, #264
-; CHECK-SD-NEXT: add x10, sp, #72
-; CHECK-SD-NEXT: ld1 { v0.b }[9], [x12]
-; CHECK-SD-NEXT: add x12, sp, #392
-; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x12]
-; CHECK-SD-NEXT: add x12, sp, #48
-; CHECK-SD-NEXT: ld1 { v1.b }[12], [x15]
-; CHECK-SD-NEXT: add x15, sp, #120
-; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v18.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v0.b }[10], [x13]
-; CHECK-SD-NEXT: ld1 { v3.b }[5], [x15]
-; CHECK-SD-NEXT: add x15, sp, #408
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11]
-; CHECK-SD-NEXT: add x13, sp, #56
-; CHECK-SD-NEXT: ld1 { v1.b }[13], [x16]
-; CHECK-SD-NEXT: add x11, sp, #64
-; CHECK-SD-NEXT: add x16, sp, #616
-; CHECK-SD-NEXT: movi v19.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v0.b }[11], [x14]
-; CHECK-SD-NEXT: add x14, sp, #600
-; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x15]
-; CHECK-SD-NEXT: add x15, sp, #608
-; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #488
-; CHECK-SD-NEXT: add x14, sp, #320
-; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x14]
-; CHECK-SD-NEXT: ld1 { v4.b }[10], [x15]
-; CHECK-SD-NEXT: add x14, sp, #624
-; CHECK-SD-NEXT: add x9, sp, #688
-; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #432
-; CHECK-SD-NEXT: add x12, sp, #328
-; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
-; CHECK-SD-NEXT: add x13, sp, #416
-; CHECK-SD-NEXT: ld1 { v2.b }[7], [x12]
-; CHECK-SD-NEXT: ld1 { v5.b }[10], [x13]
-; CHECK-SD-NEXT: ld1 { v4.b }[11], [x16]
-; CHECK-SD-NEXT: add x16, sp, #680
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x16]
-; CHECK-SD-NEXT: add x13, sp, #632
-; CHECK-SD-NEXT: add x12, sp, #504
-; CHECK-SD-NEXT: ld1 { v0.b }[14], [x11]
-; CHECK-SD-NEXT: add x11, sp, #424
-; CHECK-SD-NEXT: add x15, sp, #128
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[12], [x14]
-; CHECK-SD-NEXT: add x11, sp, #696
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v3.b }[6], [x15]
-; CHECK-SD-NEXT: add x9, sp, #640
-; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
-; CHECK-SD-NEXT: add x10, sp, #496
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT: ld1 { v4.b }[13], [x13]
-; CHECK-SD-NEXT: add x10, sp, #440
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #512
-; CHECK-SD-NEXT: add x8, sp, #136
-; CHECK-SD-NEXT: usdot v17.4s, v0.16b, v1.16b
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x10]
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x12]
-; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #448
-; CHECK-SD-NEXT: add x10, sp, #704
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
-; CHECK-SD-NEXT: add x8, sp, #648
-; CHECK-SD-NEXT: add x10, sp, #520
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #456
-; CHECK-SD-NEXT: add x9, sp, #712
-; CHECK-SD-NEXT: usdot v19.2s, v3.8b, v2.8b
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
-; CHECK-SD-NEXT: addv s0, v17.4s
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
-; CHECK-SD-NEXT: addp v1.2s, v19.2s, v19.2s
-; CHECK-SD-NEXT: fmov w8, s0
-; CHECK-SD-NEXT: usdot v16.4s, v5.16b, v4.16b
-; CHECK-SD-NEXT: usdot v18.2s, v7.8b, v6.8b
-; CHECK-SD-NEXT: fmov w9, s1
-; CHECK-SD-NEXT: addv s2, v16.4s
-; CHECK-SD-NEXT: addp v3.2s, v18.2s, v18.2s
-; CHECK-SD-NEXT: add w8, w8, w9
-; CHECK-SD-NEXT: fmov w10, s2
-; CHECK-SD-NEXT: fmov w11, s3
-; CHECK-SD-NEXT: add w9, w10, w11
-; CHECK-SD-NEXT: add w0, w8, w9
-; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_usdot_v24i8_double:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w9, [sp, #16]
-; CHECK-GI-NEXT: ldr w8, [sp, #24]
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w11, [sp, #208]
-; CHECK-GI-NEXT: ldr w13, [sp, #32]
-; CHECK-GI-NEXT: ldr w12, [sp, #144]
-; CHECK-GI-NEXT: fmov s1, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #80]
-; CHECK-GI-NEXT: ldr w10, [sp, #152]
-; CHECK-GI-NEXT: fmov s4, w11
-; CHECK-GI-NEXT: fmov s3, w12
-; CHECK-GI-NEXT: ldr w11, [sp, #160]
-; CHECK-GI-NEXT: fmov s2, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #216]
-; CHECK-GI-NEXT: ldr w14, [sp, #128]
-; CHECK-GI-NEXT: mov v1.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: ldr w12, [sp, #240]
-; CHECK-GI-NEXT: mov v4.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #96]
-; CHECK-GI-NEXT: mov v3.b[1], w10
-; CHECK-GI-NEXT: mov v2.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #40]
-; CHECK-GI-NEXT: ldr w10, [sp, #272]
-; CHECK-GI-NEXT: ldr w15, [sp, #312]
-; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: mov v1.b[2], w13
-; CHECK-GI-NEXT: fmov s5, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #48]
-; CHECK-GI-NEXT: mov v3.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #232]
-; CHECK-GI-NEXT: ldr w13, [sp, #72]
-; CHECK-GI-NEXT: mov v2.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #280]
-; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: mov v1.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #224]
-; CHECK-GI-NEXT: mov v5.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #168]
-; CHECK-GI-NEXT: mov v4.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #104]
-; CHECK-GI-NEXT: mov v3.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #64]
-; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: mov v2.b[3], w8
-; CHECK-GI-NEXT: mov v1.b[4], w10
-; CHECK-GI-NEXT: ldr w8, [sp, #288]
-; CHECK-GI-NEXT: ldr w10, [sp, #56]
-; CHECK-GI-NEXT: mov v4.b[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #112]
-; CHECK-GI-NEXT: mov v5.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #176]
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: mov v2.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #120]
-; CHECK-GI-NEXT: mov v1.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #296]
-; CHECK-GI-NEXT: mov v3.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #136]
-; CHECK-GI-NEXT: mov v4.b[4], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #248]
-; CHECK-GI-NEXT: mov v5.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #184]
-; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v2.b[5], w11
-; CHECK-GI-NEXT: mov v1.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #304]
-; CHECK-GI-NEXT: mov v3.b[5], w10
-; CHECK-GI-NEXT: ldr w11, [sp, #192]
-; CHECK-GI-NEXT: ldr w10, [sp, #200]
-; CHECK-GI-NEXT: mov v4.b[5], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #264]
-; CHECK-GI-NEXT: mov v5.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #256]
-; CHECK-GI-NEXT: mov v0.b[6], w6
-; CHECK-GI-NEXT: mov v2.b[6], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #336]
-; CHECK-GI-NEXT: mov v1.b[7], w13
-; CHECK-GI-NEXT: mov v3.b[6], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #344]
-; CHECK-GI-NEXT: ldr w13, [sp, #320]
-; CHECK-GI-NEXT: fmov s6, w14
-; CHECK-GI-NEXT: ldr w14, [sp, #400]
-; CHECK-GI-NEXT: mov v4.b[6], w9
-; CHECK-GI-NEXT: mov v5.b[5], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #408]
-; CHECK-GI-NEXT: ldr w9, [sp, #328]
-; CHECK-GI-NEXT: fmov s7, w14
-; CHECK-GI-NEXT: ldr w14, [sp, #352]
-; CHECK-GI-NEXT: mov v2.b[7], w8
-; CHECK-GI-NEXT: mov v6.b[1], w11
-; CHECK-GI-NEXT: mov v3.b[7], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #416]
-; CHECK-GI-NEXT: mov v4.b[7], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #472]
-; CHECK-GI-NEXT: ldr w11, [sp, #360]
-; CHECK-GI-NEXT: mov v7.b[1], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #464]
-; CHECK-GI-NEXT: mov v5.b[6], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #536]
-; CHECK-GI-NEXT: ldr w8, [sp, #368]
-; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v6.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #528]
-; CHECK-GI-NEXT: fmov s16, w15
-; CHECK-GI-NEXT: ldr w15, [sp, #600]
-; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: fmov s17, w14
-; CHECK-GI-NEXT: mov v7.b[2], w10
-; CHECK-GI-NEXT: ldr w14, [sp, #592]
-; CHECK-GI-NEXT: mov v16.b[1], w12
-; CHECK-GI-NEXT: mov v5.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #480]
-; CHECK-GI-NEXT: fmov s18, w14
-; CHECK-GI-NEXT: ldr w14, [sp, #544]
-; CHECK-GI-NEXT: mov v6.b[3], w11
-; CHECK-GI-NEXT: mov v17.b[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #424]
-; CHECK-GI-NEXT: ldr w11, [sp, #608]
-; CHECK-GI-NEXT: ldr w10, [sp, #376]
-; CHECK-GI-NEXT: ldr w12, [sp, #384]
-; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT: mov v7.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #656]
-; CHECK-GI-NEXT: mov v18.b[1], w15
-; CHECK-GI-NEXT: mov v16.b[2], w9
-; CHECK-GI-NEXT: mov v6.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #616]
-; CHECK-GI-NEXT: fmov s19, w13
-; CHECK-GI-NEXT: mov v17.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #664]
-; CHECK-GI-NEXT: ldr w13, [sp, #432]
-; CHECK-GI-NEXT: ushll v20.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: mov v18.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #488]
-; CHECK-GI-NEXT: sshll v21.4s, v3.4h, #0
-; CHECK-GI-NEXT: mov v19.b[1], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #552]
-; CHECK-GI-NEXT: mov v7.b[4], w13
-; CHECK-GI-NEXT: mov v16.b[3], w11
-; CHECK-GI-NEXT: ldr w13, [sp, #496]
-; CHECK-GI-NEXT: ldr w11, [sp, #440]
-; CHECK-GI-NEXT: mov v17.b[3], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #672]
-; CHECK-GI-NEXT: mov v6.b[5], w10
-; CHECK-GI-NEXT: mov v18.b[3], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #624]
-; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT: mov v19.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #560]
-; CHECK-GI-NEXT: mov v7.b[5], w11
-; CHECK-GI-NEXT: mov v16.b[4], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #680]
-; CHECK-GI-NEXT: ldr w11, [sp, #504]
-; CHECK-GI-NEXT: mov v17.b[4], w14
-; CHECK-GI-NEXT: mov v6.b[6], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #632]
-; CHECK-GI-NEXT: mov v18.b[4], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #512]
-; CHECK-GI-NEXT: ldr w9, [sp, #392]
-; CHECK-GI-NEXT: mov v19.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #568]
-; CHECK-GI-NEXT: ldr w14, [sp, #584]
-; CHECK-GI-NEXT: mov v16.b[5], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #688]
-; CHECK-GI-NEXT: sshll v4.8h, v4.8b, #0
-; CHECK-GI-NEXT: mov v17.b[5], w13
-; CHECK-GI-NEXT: ldr w15, [sp, #448]
-; CHECK-GI-NEXT: mul v20.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT: mov v18.b[5], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #576]
-; CHECK-GI-NEXT: mul v0.4s, v0.4s, v3.4s
-; CHECK-GI-NEXT: mov v19.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #640]
-; CHECK-GI-NEXT: mov v6.b[7], w9
-; CHECK-GI-NEXT: mov v16.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #696]
-; CHECK-GI-NEXT: mov v7.b[6], w15
-; CHECK-GI-NEXT: mov v17.b[6], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #648]
-; CHECK-GI-NEXT: ushll v22.4s, v1.4h, #0
-; CHECK-GI-NEXT: mov v18.b[6], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #704]
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: mov v19.b[5], w10
-; CHECK-GI-NEXT: sshll v21.4s, v4.4h, #0
-; CHECK-GI-NEXT: sshll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: ldr w8, [sp, #456]
-; CHECK-GI-NEXT: ldr w13, [sp, #520]
-; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
-; CHECK-GI-NEXT: mov v17.b[7], w14
-; CHECK-GI-NEXT: ldr w10, [sp, #712]
-; CHECK-GI-NEXT: sshll v5.8h, v5.8b, #0
-; CHECK-GI-NEXT: mov v18.b[7], w12
-; CHECK-GI-NEXT: mul v1.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT: addv s4, v20.4s
-; CHECK-GI-NEXT: mov v19.b[6], w11
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
-; CHECK-GI-NEXT: mov v7.b[7], w8
-; CHECK-GI-NEXT: mov v16.b[7], w13
-; CHECK-GI-NEXT: ushll v23.4s, v2.4h, #0
-; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
-; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT: sshll v3.4s, v5.4h, #0
-; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: mul v21.4s, v22.4s, v21.4s
-; CHECK-GI-NEXT: fmov w8, s4
-; CHECK-GI-NEXT: mov v19.b[7], w10
-; CHECK-GI-NEXT: sshll v4.8h, v18.8b, #0
-; CHECK-GI-NEXT: fmov w9, s0
-; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0
-; CHECK-GI-NEXT: sshll v18.4s, v17.4h, #0
-; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT: ushll v5.8h, v7.8b, #0
-; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0
-; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
-; CHECK-GI-NEXT: mul v3.4s, v23.4s, v3.4s
-; CHECK-GI-NEXT: sshll2 v22.4s, v4.8h, #0
-; CHECK-GI-NEXT: sshll v16.8h, v19.8b, #0
-; CHECK-GI-NEXT: mul v0.4s, v0.4s, v18.4s
-; CHECK-GI-NEXT: addv s18, v21.4s
-; CHECK-GI-NEXT: ushll2 v19.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v20.4s, v7.4h, #0
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: mul v6.4s, v6.4s, v17.4s
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
-; CHECK-GI-NEXT: sshll v23.4s, v16.4h, #0
-; CHECK-GI-NEXT: fmov w9, s18
-; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
-; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0
-; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
-; CHECK-GI-NEXT: mul v17.4s, v19.4s, v22.4s
-; CHECK-GI-NEXT: addv s3, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: mul v19.4s, v20.4s, v23.4s
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: addv s1, v6.4s
-; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s
-; CHECK-GI-NEXT: mul v5.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT: fmov w10, s3
-; CHECK-GI-NEXT: addv s3, v17.4s
-; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: addv s6, v19.4s
-; CHECK-GI-NEXT: fmov w12, s1
-; CHECK-GI-NEXT: addv s4, v4.4s
-; CHECK-GI-NEXT: addv s5, v5.4s
-; CHECK-GI-NEXT: add w9, w9, w10
-; CHECK-GI-NEXT: fmov w10, s2
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s3
-; CHECK-GI-NEXT: fmov w13, s6
-; CHECK-GI-NEXT: fmov w14, s4
-; CHECK-GI-NEXT: add w9, w9, w10
-; CHECK-GI-NEXT: add w12, w12, w13
-; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w10, w11, w14
-; CHECK-GI-NEXT: add w11, w12, w13
-; CHECK-GI-NEXT: add w9, w10, w11
-; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT: ret
-entry:
- %az = zext <24 x i8> %a to <24 x i32>
- %bz = sext <24 x i8> %b to <24 x i32>
- %m1 = mul nuw nsw <24 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)
- %cz = zext <24 x i8> %c to <24 x i32>
- %dz = sext <24 x i8> %d to <24 x i32>
- %m2 = mul nuw nsw <24 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)
- %x = add i32 %r1, %r2
- ret i32 %x
-}
-
-define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v25i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q3, q0, [x1]
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT: ushll v3.4s, v4.4h, #0
-; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT: mov v5.s[0], v3.s[0]
-; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a
- %1 = zext <25 x i8> %0 to <25 x i32>
- %2 = load <25 x i8>, ptr %b
- %3 = zext <25 x i8> %2 to <25 x i32>
- %4 = mul nuw nsw <25 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
- %op.extra = add i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v25i8_nomla:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
-; CHECK-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NEXT: ushll v4.8h, v2.8b, #0
-; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h
-; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h
-; CHECK-NEXT: mov v0.s[0], v3.s[0]
-; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h
-; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a1
- %1 = zext <25 x i8> %0 to <25 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)
- ret i32 %2
-}
-define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v25i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q3, q0, [x1]
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b
-; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b
-; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b
-; CHECK-NEXT: sshll v3.4s, v4.4h, #0
-; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT: mov v5.s[0], v3.s[0]
-; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a
- %1 = sext <25 x i8> %0 to <25 x i32>
- %2 = load <25 x i8>, ptr %b
- %3 = sext <25 x i8> %2 to <25 x i32>
- %4 = mul nsw <25 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
-; CHECK-LABEL: test_sdot_v25i8_double:
+define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
+; CHECK-LABEL: test_sdot_v25i8_double:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
@@ -3266,284 +2630,6 @@ entry:
ret i32 %x
}
-define i32 @test_usdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_usdot_v25i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q0, [x0]
-; CHECK-NEXT: ldp q5, q1, [x1]
-; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0
-; CHECK-NEXT: ushll v6.8h, v2.8b, #0
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0
-; CHECK-NEXT: sshll v7.8h, v5.8b, #0
-; CHECK-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h
-; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h
-; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h
-; CHECK-NEXT: mov v4.s[0], v3.s[0]
-; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0
-; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h
-; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h
-; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h
-; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h
-; CHECK-NEXT: add v0.4s, v6.4s, v4.4s
-; CHECK-NEXT: add v0.4s, v0.4s, v16.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <25 x i8>, ptr %a
- %1 = zext <25 x i8> %0 to <25 x i32>
- %2 = load <25 x i8>, ptr %b
- %3 = sext <25 x i8> %2 to <25 x i32>
- %4 = mul nsw <25 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_usdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {
-; CHECK-LABEL: test_usdot_v25i8_double:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: ldr b1, [sp, #16]
-; CHECK-NEXT: ldr b0, [sp, #80]
-; CHECK-NEXT: add x11, sp, #24
-; CHECK-NEXT: ldr b3, [sp, #216]
-; CHECK-NEXT: ldr b4, [sp, #152]
-; CHECK-NEXT: add x10, sp, #88
-; CHECK-NEXT: ldr b2, [sp, #280]
-; CHECK-NEXT: ld1 { v1.b }[1], [x11]
-; CHECK-NEXT: add x11, sp, #224
-; CHECK-NEXT: add x12, sp, #160
-; CHECK-NEXT: ldr b6, [sp, #480]
-; CHECK-NEXT: ld1 { v0.b }[1], [x10]
-; CHECK-NEXT: add x10, sp, #288
-; CHECK-NEXT: ld1 { v3.b }[1], [x11]
-; CHECK-NEXT: ld1 { v4.b }[1], [x12]
-; CHECK-NEXT: add x12, sp, #488
-; CHECK-NEXT: ld1 { v2.b }[1], [x10]
-; CHECK-NEXT: add x9, sp, #96
-; CHECK-NEXT: ld1 { v6.b }[1], [x12]
-; CHECK-NEXT: add x12, sp, #32
-; CHECK-NEXT: ld1 { v0.b }[2], [x9]
-; CHECK-NEXT: ld1 { v1.b }[2], [x12]
-; CHECK-NEXT: add x12, sp, #232
-; CHECK-NEXT: add x9, sp, #296
-; CHECK-NEXT: ld1 { v3.b }[2], [x12]
-; CHECK-NEXT: add x12, sp, #168
-; CHECK-NEXT: ld1 { v2.b }[2], [x9]
-; CHECK-NEXT: ld1 { v4.b }[2], [x12]
-; CHECK-NEXT: add x12, sp, #40
-; CHECK-NEXT: add x8, sp, #104
-; CHECK-NEXT: ld1 { v1.b }[3], [x12]
-; CHECK-NEXT: ld1 { v0.b }[3], [x8]
-; CHECK-NEXT: add x8, sp, #304
-; CHECK-NEXT: ld1 { v2.b }[3], [x8]
-; CHECK-NEXT: add x8, sp, #48
-; CHECK-NEXT: add x10, sp, #112
-; CHECK-NEXT: add x14, sp, #56
-; CHECK-NEXT: add x12, sp, #240
-; CHECK-NEXT: add x11, sp, #120
-; CHECK-NEXT: ld1 { v1.b }[4], [x8]
-; CHECK-NEXT: ld1 { v0.b }[4], [x10]
-; CHECK-NEXT: add x8, sp, #312
-; CHECK-NEXT: ld1 { v3.b }[3], [x12]
-; CHECK-NEXT: add x12, sp, #176
-; CHECK-NEXT: ld1 { v2.b }[4], [x8]
-; CHECK-NEXT: ld1 { v4.b }[3], [x12]
-; CHECK-NEXT: add x12, sp, #64
-; CHECK-NEXT: ldr b18, [sp, #552]
-; CHECK-NEXT: ld1 { v1.b }[5], [x14]
-; CHECK-NEXT: add x16, sp, #320
-; CHECK-NEXT: ld1 { v0.b }[5], [x11]
-; CHECK-NEXT: ldr b16, [sp, #352]
-; CHECK-NEXT: add x8, sp, #248
-; CHECK-NEXT: ld1 { v2.b }[5], [x16]
-; CHECK-NEXT: add x16, sp, #360
-; CHECK-NEXT: add x15, sp, #128
-; CHECK-NEXT: ld1 { v3.b }[4], [x8]
-; CHECK-NEXT: ld1 { v1.b }[6], [x12]
-; CHECK-NEXT: add x12, sp, #560
-; CHECK-NEXT: ld1 { v16.b }[1], [x16]
-; CHECK-NEXT: ld1 { v18.b }[1], [x12]
-; CHECK-NEXT: add x10, sp, #72
-; CHECK-NEXT: ld1 { v0.b }[6], [x15]
-; CHECK-NEXT: add x15, sp, #184
-; CHECK-NEXT: add x12, sp, #568
-; CHECK-NEXT: add x13, sp, #328
-; CHECK-NEXT: add x14, sp, #256
-; CHECK-NEXT: ld1 { v4.b }[4], [x15]
-; CHECK-NEXT: ld1 { v1.b }[7], [x10]
-; CHECK-NEXT: add x10, sp, #368
-; CHECK-NEXT: ld1 { v18.b }[2], [x12]
-; CHECK-NEXT: add x9, sp, #136
-; CHECK-NEXT: ld1 { v3.b }[5], [x14]
-; CHECK-NEXT: ld1 { v2.b }[6], [x13]
-; CHECK-NEXT: add x13, sp, #496
-; CHECK-NEXT: ld1 { v16.b }[2], [x10]
-; CHECK-NEXT: fmov s5, w0
-; CHECK-NEXT: ld1 { v0.b }[7], [x9]
-; CHECK-NEXT: add x9, sp, #192
-; CHECK-NEXT: ld1 { v6.b }[2], [x13]
-; CHECK-NEXT: add x10, sp, #576
-; CHECK-NEXT: add x11, sp, #264
-; CHECK-NEXT: ld1 { v4.b }[5], [x9]
-; CHECK-NEXT: add x9, sp, #376
-; CHECK-NEXT: ld1 { v18.b }[3], [x10]
-; CHECK-NEXT: ld1 { v3.b }[6], [x11]
-; CHECK-NEXT: add x11, sp, #504
-; CHECK-NEXT: ld1 { v16.b }[3], [x9]
-; CHECK-NEXT: mov v5.b[1], w1
-; CHECK-NEXT: ldr b7, [sp, #144]
-; CHECK-NEXT: ldr b17, [sp, #344]
-; CHECK-NEXT: add x9, sp, #200
-; CHECK-NEXT: ld1 { v6.b }[3], [x11]
-; CHECK-NEXT: add x10, sp, #584
-; CHECK-NEXT: ld1 { v4.b }[6], [x9]
-; CHECK-NEXT: add x9, sp, #384
-; CHECK-NEXT: ld1 { v18.b }[4], [x10]
-; CHECK-NEXT: ushll v7.8h, v7.8b, #0
-; CHECK-NEXT: sshll v17.8h, v17.8b, #0
-; CHECK-NEXT: add x11, sp, #512
-; CHECK-NEXT: ld1 { v16.b }[4], [x9]
-; CHECK-NEXT: mov v5.b[2], w2
-; CHECK-NEXT: ld1 { v6.b }[4], [x11]
-; CHECK-NEXT: add x11, sp, #592
-; CHECK-NEXT: add x10, sp, #392
-; CHECK-NEXT: ldr b19, [sp, #680]
-; CHECK-NEXT: ld1 { v18.b }[5], [x11]
-; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h
-; CHECK-NEXT: ldr b17, [sp, #416]
-; CHECK-NEXT: ld1 { v16.b }[5], [x10]
-; CHECK-NEXT: add x10, sp, #688
-; CHECK-NEXT: add x9, sp, #424
-; CHECK-NEXT: ld1 { v19.b }[1], [x10]
-; CHECK-NEXT: add x10, sp, #600
-; CHECK-NEXT: ldr b20, [sp, #616]
-; CHECK-NEXT: ld1 { v17.b }[1], [x9]
-; CHECK-NEXT: add x11, sp, #400
-; CHECK-NEXT: ld1 { v18.b }[6], [x10]
-; CHECK-NEXT: add x12, sp, #624
-; CHECK-NEXT: mov v5.b[3], w3
-; CHECK-NEXT: ld1 { v16.b }[6], [x11]
-; CHECK-NEXT: add x11, sp, #696
-; CHECK-NEXT: ld1 { v20.b }[1], [x12]
-; CHECK-NEXT: add x9, sp, #432
-; CHECK-NEXT: ld1 { v19.b }[2], [x11]
-; CHECK-NEXT: add x11, sp, #608
-; CHECK-NEXT: ld1 { v17.b }[2], [x9]
-; CHECK-NEXT: ld1 { v18.b }[7], [x11]
-; CHECK-NEXT: add x11, sp, #632
-; CHECK-NEXT: add x10, sp, #408
-; CHECK-NEXT: ld1 { v20.b }[2], [x11]
-; CHECK-NEXT: mov v5.b[4], w4
-; CHECK-NEXT: ld1 { v16.b }[7], [x10]
-; CHECK-NEXT: add x10, sp, #704
-; CHECK-NEXT: add x12, sp, #440
-; CHECK-NEXT: ld1 { v19.b }[3], [x10]
-; CHECK-NEXT: ld1 { v17.b }[3], [x12]
-; CHECK-NEXT: add x12, sp, #640
-; CHECK-NEXT: ld1 { v20.b }[3], [x12]
-; CHECK-NEXT: sshll v18.8h, v18.8b, #0
-; CHECK-NEXT: add x10, sp, #448
-; CHECK-NEXT: ushll v21.8h, v16.8b, #0
-; CHECK-NEXT: add x11, sp, #712
-; CHECK-NEXT: mov v5.b[5], w5
-; CHECK-NEXT: ld1 { v19.b }[4], [x11]
-; CHECK-NEXT: ld1 { v17.b }[4], [x10]
-; CHECK-NEXT: add x10, sp, #648
-; CHECK-NEXT: add x9, sp, #520
-; CHECK-NEXT: ld1 { v20.b }[4], [x10]
-; CHECK-NEXT: ldr b22, [sp, #544]
-; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h
-; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h
-; CHECK-NEXT: ldr b21, [sp, #744]
-; CHECK-NEXT: add x11, sp, #720
-; CHECK-NEXT: ld1 { v6.b }[5], [x9]
-; CHECK-NEXT: add x9, sp, #456
-; CHECK-NEXT: ld1 { v19.b }[5], [x11]
-; CHECK-NEXT: ld1 { v17.b }[5], [x9]
-; CHECK-NEXT: add x9, sp, #656
-; CHECK-NEXT: mov v5.b[6], w6
-; CHECK-NEXT: ushll v22.8h, v22.8b, #0
-; CHECK-NEXT: sshll v21.8h, v21.8b, #0
-; CHECK-NEXT: ld1 { v20.b }[5], [x9]
-; CHECK-NEXT: add x10, sp, #528
-; CHECK-NEXT: add x11, sp, #728
-; CHECK-NEXT: ld1 { v6.b }[6], [x10]
-; CHECK-NEXT: ld1 { v19.b }[6], [x11]
-; CHECK-NEXT: add x10, sp, #464
-; CHECK-NEXT: add x11, sp, #664
-; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h
-; CHECK-NEXT: movi v22.2d, #0000000000000000
-; CHECK-NEXT: ld1 { v17.b }[6], [x10]
-; CHECK-NEXT: ld1 { v20.b }[6], [x11]
-; CHECK-NEXT: mov v5.b[7], w7
-; CHECK-NEXT: add x9, sp, #536
-; CHECK-NEXT: add x10, sp, #736
-; CHECK-NEXT: add x11, sp, #208
-; CHECK-NEXT: ld1 { v6.b }[7], [x9]
-; CHECK-NEXT: ld1 { v19.b }[7], [x10]
-; CHECK-NEXT: ld1 { v4.b }[7], [x11]
-; CHECK-NEXT: add x9, sp, #472
-; CHECK-NEXT: add x10, sp, #672
-; CHECK-NEXT: add x8, sp, #336
-; CHECK-NEXT: ld1 { v17.b }[7], [x9]
-; CHECK-NEXT: ld1 { v20.b }[7], [x10]
-; CHECK-NEXT: mov v22.s[0], v21.s[0]
-; CHECK-NEXT: movi v21.2d, #0000000000000000
-; CHECK-NEXT: ushll v5.8h, v5.8b, #0
-; CHECK-NEXT: ushll v6.8h, v6.8b, #0
-; CHECK-NEXT: sshll v19.8h, v19.8b, #0
-; CHECK-NEXT: ld1 { v2.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #272
-; CHECK-NEXT: sshll v4.8h, v4.8b, #0
-; CHECK-NEXT: ld1 { v3.b }[7], [x8]
-; CHECK-NEXT: ushll v17.8h, v17.8b, #0
-; CHECK-NEXT: sshll v20.8h, v20.8b, #0
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h
-; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h
-; CHECK-NEXT: mov v21.s[0], v7.s[0]
-; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h
-; CHECK-NEXT: sshll v2.8h, v2.8b, #0
-; CHECK-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h
-; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h
-; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h
-; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h
-; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h
-; CHECK-NEXT: add v5.4s, v18.4s, v22.4s
-; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h
-; CHECK-NEXT: add v0.4s, v6.4s, v21.4s
-; CHECK-NEXT: add v2.4s, v5.4s, v16.4s
-; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: add v0.4s, v0.4s, v4.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
-entry:
- %az = zext <25 x i8> %a to <25 x i32>
- %bz = sext <25 x i8> %b to <25 x i32>
- %m1 = mul nuw nsw <25 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)
- %cz = zext <25 x i8> %c to <25 x i32>
- %dz = sext <25 x i8> %d to <25 x i32>
- %m2 = mul nuw nsw <25 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)
- %x = add i32 %r1, %r2
- ret i32 %x
-}
-
define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
; CHECK-SD-LABEL: test_udot_v32i8:
; CHECK-SD: // %bb.0: // %entry
@@ -4371,2941 +3457,1507 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %
; CHECK-NEXT: add x12, sp, #616
; CHECK-NEXT: ldr b16, [sp, #544]
; CHECK-NEXT: ld1 { v0.b }[4], [x10]
-; CHECK-NEXT: add x10, sp, #48
-; CHECK-NEXT: ld1 { v4.b }[1], [x12]
-; CHECK-NEXT: add x12, sp, #176
-; CHECK-NEXT: ld1 { v5.b }[2], [x13]
-; CHECK-NEXT: add x13, sp, #680
-; CHECK-NEXT: ld1 { v3.b }[4], [x10]
-; CHECK-NEXT: ld1 { v2.b }[4], [x12]
-; CHECK-NEXT: ld1 { v6.b }[1], [x13]
-; CHECK-NEXT: add x13, sp, #56
-; CHECK-NEXT: ld1 { v0.b }[5], [x9]
-; CHECK-NEXT: mov v1.b[2], w2
-; CHECK-NEXT: add x8, sp, #128
-; CHECK-NEXT: add x14, sp, #184
-; CHECK-NEXT: add x11, sp, #136
-; CHECK-NEXT: ld1 { v3.b }[5], [x13]
-; CHECK-NEXT: add x13, sp, #552
-; CHECK-NEXT: ld1 { v2.b }[5], [x14]
-; CHECK-NEXT: ld1 { v16.b }[1], [x13]
-; CHECK-NEXT: add x14, sp, #624
-; CHECK-NEXT: ld1 { v0.b }[6], [x8]
-; CHECK-NEXT: add x8, sp, #688
-; CHECK-NEXT: add x13, sp, #504
-; CHECK-NEXT: ld1 { v4.b }[2], [x14]
-; CHECK-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-NEXT: add x8, sp, #560
-; CHECK-NEXT: ld1 { v5.b }[3], [x13]
-; CHECK-NEXT: ld1 { v16.b }[2], [x8]
-; CHECK-NEXT: mov v1.b[3], w3
-; CHECK-NEXT: add x9, sp, #64
-; CHECK-NEXT: add x15, sp, #632
-; CHECK-NEXT: ld1 { v3.b }[6], [x9]
-; CHECK-NEXT: ld1 { v0.b }[7], [x11]
-; CHECK-NEXT: ld1 { v4.b }[3], [x15]
-; CHECK-NEXT: add x8, sp, #696
-; CHECK-NEXT: add x9, sp, #568
-; CHECK-NEXT: add x11, sp, #512
-; CHECK-NEXT: ld1 { v6.b }[3], [x8]
-; CHECK-NEXT: ld1 { v16.b }[3], [x9]
-; CHECK-NEXT: ld1 { v5.b }[4], [x11]
-; CHECK-NEXT: add x8, sp, #640
-; CHECK-NEXT: mov v1.b[4], w4
-; CHECK-NEXT: ld1 { v4.b }[4], [x8]
-; CHECK-NEXT: add x8, sp, #704
-; CHECK-NEXT: add x9, sp, #576
-; CHECK-NEXT: add x11, sp, #520
-; CHECK-NEXT: ld1 { v6.b }[4], [x8]
-; CHECK-NEXT: ld1 { v16.b }[4], [x9]
-; CHECK-NEXT: ld1 { v5.b }[5], [x11]
-; CHECK-NEXT: ldr b18, [sp, #736]
-; CHECK-NEXT: add x12, sp, #192
-; CHECK-NEXT: ld1 { v2.b }[6], [x12]
-; CHECK-NEXT: add x8, sp, #648
-; CHECK-NEXT: add x9, sp, #528
-; CHECK-NEXT: add x11, sp, #712
-; CHECK-NEXT: add x12, sp, #584
-; CHECK-NEXT: sshll v18.8h, v18.8b, #0
-; CHECK-NEXT: mov v1.b[5], w5
-; CHECK-NEXT: ld1 { v6.b }[5], [x11]
-; CHECK-NEXT: ld1 { v16.b }[5], [x12]
-; CHECK-NEXT: ld1 { v4.b }[5], [x8]
-; CHECK-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-NEXT: movi v17.2d, #0000000000000000
-; CHECK-NEXT: add x8, sp, #656
-; CHECK-NEXT: add x9, sp, #536
-; CHECK-NEXT: add x11, sp, #720
-; CHECK-NEXT: add x12, sp, #592
-; CHECK-NEXT: sshll v18.4s, v18.4h, #0
-; CHECK-NEXT: ldr b7, [sp, #208]
-; CHECK-NEXT: ld1 { v6.b }[6], [x11]
-; CHECK-NEXT: ld1 { v16.b }[6], [x12]
-; CHECK-NEXT: ld1 { v4.b }[6], [x8]
-; CHECK-NEXT: ld1 { v5.b }[7], [x9]
-; CHECK-NEXT: mov v1.b[6], w6
-; CHECK-NEXT: sshll v7.8h, v7.8b, #0
-; CHECK-NEXT: add x8, sp, #664
-; CHECK-NEXT: add x9, sp, #728
-; CHECK-NEXT: add x11, sp, #600
-; CHECK-NEXT: mov v17.s[0], v18.s[0]
-; CHECK-NEXT: ld1 { v6.b }[7], [x9]
-; CHECK-NEXT: ld1 { v16.b }[7], [x11]
-; CHECK-NEXT: ld1 { v4.b }[7], [x8]
-; CHECK-NEXT: sshll v5.8h, v5.8b, #0
-; CHECK-NEXT: movi v18.2d, #0000000000000000
-; CHECK-NEXT: add x10, sp, #200
-; CHECK-NEXT: mov v1.b[7], w7
-; CHECK-NEXT: add x9, sp, #72
-; CHECK-NEXT: sshll v7.4s, v7.4h, #0
-; CHECK-NEXT: ld1 { v2.b }[7], [x10]
-; CHECK-NEXT: ld1 { v3.b }[7], [x9]
-; CHECK-NEXT: sshll v6.8h, v6.8b, #0
-; CHECK-NEXT: sshll v16.8h, v16.8b, #0
-; CHECK-NEXT: sshll v4.8h, v4.8b, #0
-; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h
-; CHECK-NEXT: sshll v0.8h, v0.8b, #0
-; CHECK-NEXT: mov v18.s[0], v7.s[0]
-; CHECK-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NEXT: sshll v2.8h, v2.8b, #0
-; CHECK-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-NEXT: saddl2 v7.4s, v16.8h, v6.8h
-; CHECK-NEXT: saddl2 v5.4s, v5.8h, v4.8h
-; CHECK-NEXT: saddl v6.4s, v16.4h, v6.4h
-; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h
-; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h
-; CHECK-NEXT: saddl2 v16.4s, v3.8h, v2.8h
-; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h
-; CHECK-NEXT: add v5.4s, v5.4s, v7.4s
-; CHECK-NEXT: add v4.4s, v4.4s, v6.4s
-; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h
-; CHECK-NEXT: add v6.4s, v17.4s, v16.4s
-; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT: add v1.4s, v4.4s, v5.4s
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: add v1.4s, v6.4s, v1.4s
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
-entry:
- %az = sext <33 x i8> %a to <33 x i32>
- %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)
- %cz = sext <33 x i8> %c to <33 x i32>
- %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz)
- %x = add i32 %r1, %r2
- ret i32 %x
-}
-
-define i32 @test_usdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_usdot_v33i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr b0, [x0, #32]
-; CHECK-NEXT: ldr b1, [x1, #32]
-; CHECK-NEXT: ldp q2, q4, [x0]
-; CHECK-NEXT: ldp q3, q6, [x1]
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-NEXT: ushll v5.8h, v2.8b, #0
-; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT: ushll v16.8h, v4.8b, #0
-; CHECK-NEXT: sshll v7.8h, v3.8b, #0
-; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0
-; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: sshll v19.8h, v6.8b, #0
-; CHECK-NEXT: sshll2 v6.8h, v6.16b, #0
-; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h
-; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h
-; CHECK-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h
-; CHECK-NEXT: smlal2 v18.4s, v6.8h, v4.8h
-; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h
-; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h
-; CHECK-NEXT: smlal v0.4s, v6.4h, v4.4h
-; CHECK-NEXT: add v2.4s, v17.4s, v18.4s
-; CHECK-NEXT: smlal v1.4s, v19.4h, v16.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
-entry:
- %0 = load <33 x i8>, ptr %a
- %1 = zext <33 x i8> %0 to <33 x i32>
- %2 = load <33 x i8>, ptr %b
- %3 = sext <33 x i8> %2 to <33 x i32>
- %4 = mul nsw <33 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_usdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {
-; CHECK-LABEL: test_usdot_v33i8_double:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: fmov s4, w0
-; CHECK-NEXT: ldr b0, [sp, #144]
-; CHECK-NEXT: add x8, sp, #152
-; CHECK-NEXT: ldr b1, [sp, #80]
-; CHECK-NEXT: add x10, sp, #88
-; CHECK-NEXT: ldr b6, [sp, #16]
-; CHECK-NEXT: ld1 { v0.b }[1], [x8]
-; CHECK-NEXT: ldr b2, [sp, #408]
-; CHECK-NEXT: add x9, sp, #160
-; CHECK-NEXT: mov v4.b[1], w1
-; CHECK-NEXT: ld1 { v1.b }[1], [x10]
-; CHECK-NEXT: add x10, sp, #24
-; CHECK-NEXT: ld1 { v6.b }[1], [x10]
-; CHECK-NEXT: add x10, sp, #416
-; CHECK-NEXT: add x8, sp, #168
-; CHECK-NEXT: ld1 { v0.b }[2], [x9]
-; CHECK-NEXT: add x9, sp, #96
-; CHECK-NEXT: ld1 { v2.b }[1], [x10]
-; CHECK-NEXT: ld1 { v1.b }[2], [x9]
-; CHECK-NEXT: add x9, sp, #32
-; CHECK-NEXT: add x10, sp, #424
-; CHECK-NEXT: mov v4.b[2], w2
-; CHECK-NEXT: ld1 { v6.b }[2], [x9]
-; CHECK-NEXT: add x12, sp, #176
-; CHECK-NEXT: ld1 { v2.b }[2], [x10]
-; CHECK-NEXT: add x10, sp, #104
-; CHECK-NEXT: ld1 { v0.b }[3], [x8]
-; CHECK-NEXT: ld1 { v1.b }[3], [x10]
-; CHECK-NEXT: add x10, sp, #40
-; CHECK-NEXT: add x13, sp, #112
-; CHECK-NEXT: ld1 { v6.b }[3], [x10]
-; CHECK-NEXT: add x10, sp, #432
-; CHECK-NEXT: add x11, sp, #184
-; CHECK-NEXT: mov v4.b[3], w3
-; CHECK-NEXT: ld1 { v0.b }[4], [x12]
-; CHECK-NEXT: add x12, sp, #48
-; CHECK-NEXT: ld1 { v2.b }[3], [x10]
-; CHECK-NEXT: ld1 { v1.b }[4], [x13]
-; CHECK-NEXT: add x14, sp, #440
-; CHECK-NEXT: ld1 { v6.b }[4], [x12]
-; CHECK-NEXT: ldr b5, [sp, #216]
-; CHECK-NEXT: add x10, sp, #120
-; CHECK-NEXT: ld1 { v0.b }[5], [x11]
-; CHECK-NEXT: add x11, sp, #56
-; CHECK-NEXT: ldr b7, [sp, #280]
-; CHECK-NEXT: mov v4.b[4], w4
-; CHECK-NEXT: ld1 { v2.b }[4], [x14]
-; CHECK-NEXT: add x14, sp, #224
-; CHECK-NEXT: ld1 { v6.b }[5], [x11]
-; CHECK-NEXT: ld1 { v5.b }[1], [x14]
-; CHECK-NEXT: add x15, sp, #288
-; CHECK-NEXT: ld1 { v1.b }[5], [x10]
-; CHECK-NEXT: add x14, sp, #64
-; CHECK-NEXT: ld1 { v7.b }[1], [x15]
-; CHECK-NEXT: add x9, sp, #192
-; CHECK-NEXT: add x12, sp, #128
-; CHECK-NEXT: ldr b3, [sp, #344]
-; CHECK-NEXT: mov v4.b[5], w5
-; CHECK-NEXT: ld1 { v6.b }[6], [x14]
-; CHECK-NEXT: add x14, sp, #232
-; CHECK-NEXT: ld1 { v0.b }[6], [x9]
-; CHECK-NEXT: ld1 { v5.b }[2], [x14]
-; CHECK-NEXT: ld1 { v1.b }[6], [x12]
-; CHECK-NEXT: add x12, sp, #296
-; CHECK-NEXT: add x13, sp, #352
-; CHECK-NEXT: add x8, sp, #200
-; CHECK-NEXT: ld1 { v7.b }[2], [x12]
-; CHECK-NEXT: ld1 { v3.b }[1], [x13]
-; CHECK-NEXT: add x14, sp, #240
-; CHECK-NEXT: mov v4.b[6], w6
-; CHECK-NEXT: add x10, sp, #448
-; CHECK-NEXT: ld1 { v5.b }[3], [x14]
-; CHECK-NEXT: ld1 { v0.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #304
-; CHECK-NEXT: add x9, sp, #360
-; CHECK-NEXT: ld1 { v2.b }[5], [x10]
-; CHECK-NEXT: ld1 { v7.b }[3], [x8]
-; CHECK-NEXT: add x11, sp, #72
-; CHECK-NEXT: ld1 { v3.b }[2], [x9]
-; CHECK-NEXT: add x8, sp, #248
-; CHECK-NEXT: add x15, sp, #456
-; CHECK-NEXT: mov v4.b[7], w7
-; CHECK-NEXT: ld1 { v6.b }[7], [x11]
-; CHECK-NEXT: ld1 { v5.b }[4], [x8]
-; CHECK-NEXT: add x8, sp, #312
-; CHECK-NEXT: ld1 { v2.b }[6], [x15]
-; CHECK-NEXT: add x9, sp, #368
-; CHECK-NEXT: ld1 { v7.b }[4], [x8]
-; CHECK-NEXT: ld1 { v3.b }[3], [x9]
-; CHECK-NEXT: add x8, sp, #256
-; CHECK-NEXT: ushll v17.8h, v6.8b, #0
-; CHECK-NEXT: add x10, sp, #464
-; CHECK-NEXT: ld1 { v5.b }[5], [x8]
-; CHECK-NEXT: ushll v21.8h, v4.8b, #0
-; CHECK-NEXT: ldr b4, [sp, #208]
-; CHECK-NEXT: add x8, sp, #320
-; CHECK-NEXT: ld1 { v2.b }[7], [x10]
-; CHECK-NEXT: add x9, sp, #376
-; CHECK-NEXT: ld1 { v7.b }[5], [x8]
-; CHECK-NEXT: ushll v6.8h, v4.8b, #0
-; CHECK-NEXT: ldr b4, [sp, #672]
-; CHECK-NEXT: add x10, sp, #680
-; CHECK-NEXT: ld1 { v3.b }[4], [x9]
-; CHECK-NEXT: add x9, sp, #264
-; CHECK-NEXT: ldr b16, [sp, #472]
-; CHECK-NEXT: ld1 { v4.b }[1], [x10]
-; CHECK-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-NEXT: add x9, sp, #328
-; CHECK-NEXT: ld1 { v7.b }[6], [x9]
-; CHECK-NEXT: add x10, sp, #688
-; CHECK-NEXT: add x9, sp, #272
-; CHECK-NEXT: add x8, sp, #384
-; CHECK-NEXT: sshll v16.8h, v16.8b, #0
-; CHECK-NEXT: ldr b19, [sp, #480]
-; CHECK-NEXT: ld1 { v4.b }[2], [x10]
-; CHECK-NEXT: ld1 { v5.b }[7], [x9]
-; CHECK-NEXT: add x9, sp, #336
-; CHECK-NEXT: ld1 { v3.b }[5], [x8]
-; CHECK-NEXT: ld1 { v7.b }[7], [x9]
-; CHECK-NEXT: add x9, sp, #696
-; CHECK-NEXT: add x8, sp, #392
-; CHECK-NEXT: smull v20.4s, v6.4h, v16.4h
-; CHECK-NEXT: ldr b16, [sp, #608]
-; CHECK-NEXT: ld1 { v4.b }[3], [x9]
-; CHECK-NEXT: add x9, sp, #488
-; CHECK-NEXT: sshll v22.8h, v5.8b, #0
-; CHECK-NEXT: ld1 { v19.b }[1], [x9]
-; CHECK-NEXT: ld1 { v3.b }[6], [x8]
-; CHECK-NEXT: add x9, sp, #704
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: add x10, sp, #496
-; CHECK-NEXT: add x8, sp, #400
-; CHECK-NEXT: ld1 { v4.b }[4], [x9]
-; CHECK-NEXT: add x9, sp, #616
-; CHECK-NEXT: sshll v18.8h, v7.8b, #0
-; CHECK-NEXT: ld1 { v16.b }[1], [x9]
-; CHECK-NEXT: ld1 { v19.b }[2], [x10]
-; CHECK-NEXT: ld1 { v3.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #712
-; CHECK-NEXT: add x9, sp, #504
-; CHECK-NEXT: smull2 v7.4s, v21.8h, v22.8h
-; CHECK-NEXT: ld1 { v4.b }[5], [x8]
-; CHECK-NEXT: add x8, sp, #624
-; CHECK-NEXT: mov v5.s[0], v20.s[0]
-; CHECK-NEXT: ld1 { v16.b }[2], [x8]
-; CHECK-NEXT: ld1 { v19.b }[3], [x9]
-; CHECK-NEXT: add x8, sp, #720
-; CHECK-NEXT: add x9, sp, #512
-; CHECK-NEXT: ldr b20, [sp, #544]
-; CHECK-NEXT: add x10, sp, #640
-; CHECK-NEXT: ld1 { v4.b }[6], [x8]
-; CHECK-NEXT: add x8, sp, #632
-; CHECK-NEXT: add x11, sp, #520
-; CHECK-NEXT: ld1 { v16.b }[3], [x8]
-; CHECK-NEXT: ld1 { v19.b }[4], [x9]
-; CHECK-NEXT: add x9, sp, #552
-; CHECK-NEXT: smlal v5.4s, v21.4h, v22.4h
-; CHECK-NEXT: ld1 { v20.b }[1], [x9]
-; CHECK-NEXT: ldr b21, [sp, #736]
-; CHECK-NEXT: ldr b22, [sp, #1000]
-; CHECK-NEXT: add x9, sp, #528
-; CHECK-NEXT: ldr b23, [sp, #808]
-; CHECK-NEXT: ld1 { v16.b }[4], [x10]
-; CHECK-NEXT: ld1 { v19.b }[5], [x11]
-; CHECK-NEXT: add x10, sp, #560
-; CHECK-NEXT: ushll v21.8h, v21.8b, #0
-; CHECK-NEXT: sshll v24.8h, v22.8b, #0
-; CHECK-NEXT: ld1 { v20.b }[2], [x10]
-; CHECK-NEXT: smull v6.4s, v17.4h, v18.4h
-; CHECK-NEXT: smull2 v18.4s, v17.8h, v18.8h
-; CHECK-NEXT: movi v17.2d, #0000000000000000
-; CHECK-NEXT: ld1 { v19.b }[6], [x9]
-; CHECK-NEXT: add x9, sp, #568
-; CHECK-NEXT: ldr b22, [sp, #744]
-; CHECK-NEXT: add x11, sp, #816
-; CHECK-NEXT: smull v24.4s, v21.4h, v24.4h
-; CHECK-NEXT: ld1 { v20.b }[3], [x9]
-; CHECK-NEXT: add x10, sp, #752
-; CHECK-NEXT: ld1 { v23.b }[1], [x11]
-; CHECK-NEXT: add x8, sp, #728
-; CHECK-NEXT: ld1 { v22.b }[1], [x10]
-; CHECK-NEXT: add x9, sp, #576
-; CHECK-NEXT: ldr b21, [sp, #936]
-; CHECK-NEXT: add x10, sp, #824
-; CHECK-NEXT: ld1 { v4.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #648
-; CHECK-NEXT: ld1 { v20.b }[4], [x9]
-; CHECK-NEXT: add x9, sp, #760
-; CHECK-NEXT: add x11, sp, #944
-; CHECK-NEXT: ld1 { v23.b }[2], [x10]
-; CHECK-NEXT: mov v17.s[0], v24.s[0]
-; CHECK-NEXT: ldr b24, [sp, #872]
-; CHECK-NEXT: ld1 { v16.b }[5], [x8]
-; CHECK-NEXT: add x8, sp, #536
-; CHECK-NEXT: ld1 { v22.b }[2], [x9]
-; CHECK-NEXT: ld1 { v21.b }[1], [x11]
-; CHECK-NEXT: add x10, sp, #880
-; CHECK-NEXT: ld1 { v19.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #584
-; CHECK-NEXT: add x9, sp, #832
-; CHECK-NEXT: ld1 { v24.b }[1], [x10]
-; CHECK-NEXT: ld1 { v20.b }[5], [x8]
-; CHECK-NEXT: add x8, sp, #768
-; CHECK-NEXT: ld1 { v23.b }[3], [x9]
-; CHECK-NEXT: add x9, sp, #952
-; CHECK-NEXT: ld1 { v22.b }[3], [x8]
-; CHECK-NEXT: add x11, sp, #888
-; CHECK-NEXT: ld1 { v21.b }[2], [x9]
-; CHECK-NEXT: add x8, sp, #592
-; CHECK-NEXT: add x10, sp, #840
-; CHECK-NEXT: ld1 { v24.b }[2], [x11]
-; CHECK-NEXT: add x9, sp, #776
-; CHECK-NEXT: ld1 { v23.b }[4], [x10]
-; CHECK-NEXT: ld1 { v20.b }[6], [x8]
-; CHECK-NEXT: add x8, sp, #960
-; CHECK-NEXT: ld1 { v22.b }[4], [x9]
-; CHECK-NEXT: ld1 { v21.b }[3], [x8]
-; CHECK-NEXT: add x10, sp, #896
-; CHECK-NEXT: add x9, sp, #848
-; CHECK-NEXT: ld1 { v24.b }[3], [x10]
-; CHECK-NEXT: add x8, sp, #784
-; CHECK-NEXT: ld1 { v23.b }[5], [x9]
-; CHECK-NEXT: add x9, sp, #968
-; CHECK-NEXT: ld1 { v22.b }[5], [x8]
-; CHECK-NEXT: add x11, sp, #904
-; CHECK-NEXT: ld1 { v21.b }[4], [x9]
-; CHECK-NEXT: add x8, sp, #600
-; CHECK-NEXT: add x10, sp, #856
-; CHECK-NEXT: ld1 { v24.b }[4], [x11]
-; CHECK-NEXT: add x9, sp, #792
-; CHECK-NEXT: ld1 { v23.b }[6], [x10]
-; CHECK-NEXT: ld1 { v20.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #976
-; CHECK-NEXT: ld1 { v22.b }[6], [x9]
-; CHECK-NEXT: ld1 { v21.b }[5], [x8]
-; CHECK-NEXT: add x10, sp, #912
-; CHECK-NEXT: add x9, sp, #864
-; CHECK-NEXT: ld1 { v24.b }[5], [x10]
-; CHECK-NEXT: add x8, sp, #800
-; CHECK-NEXT: ld1 { v23.b }[7], [x9]
-; CHECK-NEXT: add x9, sp, #984
-; CHECK-NEXT: ld1 { v22.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #656
-; CHECK-NEXT: ld1 { v21.b }[6], [x9]
-; CHECK-NEXT: add x9, sp, #920
-; CHECK-NEXT: ld1 { v16.b }[6], [x8]
-; CHECK-NEXT: ld1 { v24.b }[6], [x9]
-; CHECK-NEXT: add x8, sp, #992
-; CHECK-NEXT: add x13, sp, #136
-; CHECK-NEXT: ushll v19.8h, v19.8b, #0
-; CHECK-NEXT: ushll v20.8h, v20.8b, #0
-; CHECK-NEXT: sshll v22.8h, v22.8b, #0
-; CHECK-NEXT: sshll v23.8h, v23.8b, #0
-; CHECK-NEXT: add x9, sp, #664
-; CHECK-NEXT: ld1 { v21.b }[7], [x8]
-; CHECK-NEXT: add x8, sp, #928
-; CHECK-NEXT: ld1 { v1.b }[7], [x13]
-; CHECK-NEXT: ld1 { v16.b }[7], [x9]
-; CHECK-NEXT: ld1 { v24.b }[7], [x8]
-; CHECK-NEXT: smlal v17.4s, v19.4h, v22.4h
-; CHECK-NEXT: smull2 v19.4s, v19.8h, v22.8h
-; CHECK-NEXT: smull v22.4s, v20.4h, v23.4h
-; CHECK-NEXT: smull2 v20.4s, v20.8h, v23.8h
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NEXT: sshll v3.8h, v3.8b, #0
-; CHECK-NEXT: sshll v2.8h, v2.8b, #0
-; CHECK-NEXT: ushll v4.8h, v4.8b, #0
-; CHECK-NEXT: sshll v21.8h, v21.8b, #0
-; CHECK-NEXT: ushll v16.8h, v16.8b, #0
-; CHECK-NEXT: sshll v23.8h, v24.8b, #0
-; CHECK-NEXT: smlal2 v7.4s, v1.8h, v3.8h
-; CHECK-NEXT: smlal v5.4s, v1.4h, v3.4h
-; CHECK-NEXT: smlal2 v18.4s, v0.8h, v2.8h
-; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h
-; CHECK-NEXT: smlal2 v20.4s, v4.8h, v21.8h
-; CHECK-NEXT: smlal v22.4s, v4.4h, v21.4h
-; CHECK-NEXT: smlal2 v19.4s, v16.8h, v23.8h
-; CHECK-NEXT: smlal v17.4s, v16.4h, v23.4h
-; CHECK-NEXT: add v0.4s, v7.4s, v18.4s
-; CHECK-NEXT: add v1.4s, v5.4s, v6.4s
-; CHECK-NEXT: add v2.4s, v19.4s, v20.4s
-; CHECK-NEXT: add v3.4s, v17.4s, v22.4s
-; CHECK-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
-entry:
- %az = zext <33 x i8> %a to <33 x i32>
- %bz = sext <33 x i8> %b to <33 x i32>
- %m1 = mul nuw nsw <33 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1)
- %cz = zext <33 x i8> %c to <33 x i32>
- %dz = sext <33 x i8> %d to <33 x i32>
- %m2 = mul nuw nsw <33 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2)
- %x = add i32 %r1, %r2
- ret i32 %x
-}
-
-define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_udot_v48i8:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q1, [x0, #32]
-; CHECK-SD-NEXT: ldr q2, [x1, #32]
-; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-SD-NEXT: ldp q3, q1, [x0]
-; CHECK-SD-NEXT: ldp q4, q2, [x1]
-; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b
-; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
-; CHECK-SD-NEXT: addv s0, v0.4s
-; CHECK-SD-NEXT: fmov w8, s0
-; CHECK-SD-NEXT: add w0, w8, w2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_udot_v48i8:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT: ldr q7, [x0, #32]
-; CHECK-GI-NEXT: ldp q3, q4, [x0]
-; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT: ldp q5, q6, [x1]
-; CHECK-GI-NEXT: ldr q16, [x1, #32]
-; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b
-; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b
-; CHECK-GI-NEXT: udot v2.4s, v16.16b, v7.16b
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w0, w8, w2
-; CHECK-GI-NEXT: ret
-entry:
- %0 = load <48 x i8>, ptr %a
- %1 = zext <48 x i8> %0 to <48 x i32>
- %2 = load <48 x i8>, ptr %b
- %3 = zext <48 x i8> %2 to <48 x i32>
- %4 = mul nuw nsw <48 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
- %op.extra = add i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-SD-LABEL: test_udot_v48i8_nomla:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.16b, #1
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q2, [x0, #32]
-; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-SD-NEXT: ldp q3, q2, [x0]
-; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
-; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
-; CHECK-SD-NEXT: addv s0, v1.4s
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_udot_v48i8_nomla:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: movi v0.16b, #1
-; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT: ldr q6, [x0, #32]
-; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT: ldp q4, q5, [x0]
-; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
-; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b
-; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b
-; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b
-; CHECK-GI-NEXT: addv s0, v1.4s
-; CHECK-GI-NEXT: addv s1, v2.4s
-; CHECK-GI-NEXT: addv s2, v3.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ret
-entry:
- %0 = load <48 x i8>, ptr %a1
- %1 = zext <48 x i8> %0 to <48 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1)
- ret i32 %2
-}
-define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_sdot_v48i8:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q1, [x0, #32]
-; CHECK-SD-NEXT: ldr q2, [x1, #32]
-; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-SD-NEXT: ldp q3, q1, [x0]
-; CHECK-SD-NEXT: ldp q4, q2, [x1]
-; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b
-; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
-; CHECK-SD-NEXT: addv s0, v0.4s
-; CHECK-SD-NEXT: fmov w8, s0
-; CHECK-SD-NEXT: add w0, w8, w2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_sdot_v48i8:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT: ldr q7, [x0, #32]
-; CHECK-GI-NEXT: ldp q3, q4, [x0]
-; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
-; CHECK-GI-NEXT: ldp q5, q6, [x1]
-; CHECK-GI-NEXT: ldr q16, [x1, #32]
-; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b
-; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b
-; CHECK-GI-NEXT: sdot v2.4s, v16.16b, v7.16b
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w0, w8, w2
-; CHECK-GI-NEXT: ret
-entry:
- %0 = load <48 x i8>, ptr %a
- %1 = sext <48 x i8> %0 to <48 x i32>
- %2 = load <48 x i8>, ptr %b
- %3 = sext <48 x i8> %2 to <48 x i32>
- %4 = mul nsw <48 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-SD-LABEL: test_sdot_v48i8_double:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: .cfi_offset w29, -16
-; CHECK-SD-NEXT: ldr b3, [sp, #592]
-; CHECK-SD-NEXT: add x8, sp, #600
-; CHECK-SD-NEXT: ldr b6, [sp, #208]
-; CHECK-SD-NEXT: ldr b0, [sp, #336]
-; CHECK-SD-NEXT: add x9, sp, #344
-; CHECK-SD-NEXT: ldr b2, [sp, #464]
-; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #216
-; CHECK-SD-NEXT: add x10, sp, #624
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #608
-; CHECK-SD-NEXT: ld1 { v0.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #232
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: ldr b7, [sp, #1360]
-; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #224
-; CHECK-SD-NEXT: add x11, sp, #648
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #616
-; CHECK-SD-NEXT: add x12, sp, #376
-; CHECK-SD-NEXT: mov v1.b[1], w1
-; CHECK-SD-NEXT: ldr b16, [sp, #976]
-; CHECK-SD-NEXT: add x14, sp, #288
-; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #632
-; CHECK-SD-NEXT: add x15, sp, #408
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #472
-; CHECK-SD-NEXT: add x13, sp, #696
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #240
-; CHECK-SD-NEXT: add x16, sp, #448
-; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #352
-; CHECK-SD-NEXT: mov v1.b[2], w2
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v0.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1368
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #248
-; CHECK-SD-NEXT: add x9, sp, #640
-; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #656
-; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
-; CHECK-SD-NEXT: add x10, sp, #360
-; CHECK-SD-NEXT: mov v1.b[3], w3
-; CHECK-SD-NEXT: ld1 { v0.b }[3], [x10]
-; CHECK-SD-NEXT: add x10, sp, #256
-; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
-; CHECK-SD-NEXT: add x9, sp, #368
-; CHECK-SD-NEXT: ldr b17, [sp, #720]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #984
-; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #664
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11]
-; CHECK-SD-NEXT: add x11, sp, #264
-; CHECK-SD-NEXT: mov v1.b[4], w4
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11]
-; CHECK-SD-NEXT: add x9, sp, #672
-; CHECK-SD-NEXT: add x11, sp, #680
-; CHECK-SD-NEXT: ld1 { v0.b }[5], [x12]
-; CHECK-SD-NEXT: add x12, sp, #480
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
-; CHECK-SD-NEXT: add x12, sp, #272
-; CHECK-SD-NEXT: ld1 { v3.b }[8], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x12]
-; CHECK-SD-NEXT: add x12, sp, #384
-; CHECK-SD-NEXT: mov v1.b[5], w5
-; CHECK-SD-NEXT: ld1 { v0.b }[6], [x12]
-; CHECK-SD-NEXT: add x12, sp, #280
-; CHECK-SD-NEXT: add x8, sp, #688
-; CHECK-SD-NEXT: ld1 { v3.b }[9], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1376
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #392
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12]
-; CHECK-SD-NEXT: ld1 { v0.b }[7], [x10]
-; CHECK-SD-NEXT: mov v1.b[6], w6
-; CHECK-SD-NEXT: add x12, sp, #704
-; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
-; CHECK-SD-NEXT: add x9, sp, #400
-; CHECK-SD-NEXT: add x10, sp, #712
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x14]
-; CHECK-SD-NEXT: add x14, sp, #992
-; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x14]
-; CHECK-SD-NEXT: add x14, sp, #296
-; CHECK-SD-NEXT: ld1 { v3.b }[11], [x11]
-; CHECK-SD-NEXT: add x9, sp, #304
-; CHECK-SD-NEXT: add x11, sp, #312
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x14]
-; CHECK-SD-NEXT: mov v1.b[7], w7
-; CHECK-SD-NEXT: add x14, sp, #320
-; CHECK-SD-NEXT: ld1 { v0.b }[9], [x15]
-; CHECK-SD-NEXT: add x15, sp, #328
-; CHECK-SD-NEXT: ld1 { v3.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #416
-; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1384
-; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #424
-; CHECK-SD-NEXT: ld1 { v3.b }[13], [x13]
-; CHECK-SD-NEXT: add x8, sp, #432
-; CHECK-SD-NEXT: add x13, sp, #440
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11]
-; CHECK-SD-NEXT: add x11, sp, #16
-; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1000
-; CHECK-SD-NEXT: ld1 { v1.b }[8], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v3.b }[14], [x12]
-; CHECK-SD-NEXT: add x12, sp, #488
-; CHECK-SD-NEXT: ld1 { v6.b }[14], [x14]
-; CHECK-SD-NEXT: add x14, sp, #1392
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12]
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
-; CHECK-SD-NEXT: add x11, sp, #1008
-; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8]
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1400
-; CHECK-SD-NEXT: ld1 { v3.b }[15], [x10]
-; CHECK-SD-NEXT: add x10, sp, #496
-; CHECK-SD-NEXT: add x9, sp, #24
-; CHECK-SD-NEXT: ld1 { v6.b }[15], [x15]
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1016
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
-; CHECK-SD-NEXT: add x8, sp, #1408
-; CHECK-SD-NEXT: ld1 { v1.b }[9], [x9]
-; CHECK-SD-NEXT: add x9, sp, #504
-; CHECK-SD-NEXT: add x10, sp, #512
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1024
-; CHECK-SD-NEXT: add x8, sp, #32
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v0.b }[14], [x16]
-; CHECK-SD-NEXT: ld1 { v1.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1416
-; CHECK-SD-NEXT: add x9, sp, #456
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1032
-; CHECK-SD-NEXT: add x8, sp, #40
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v0.b }[15], [x9]
-; CHECK-SD-NEXT: ld1 { v1.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1424
-; CHECK-SD-NEXT: add x9, sp, #520
-; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1040
-; CHECK-SD-NEXT: add x8, sp, #48
-; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9]
-; CHECK-SD-NEXT: add x10, sp, #528
-; CHECK-SD-NEXT: ld1 { v1.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1432
-; CHECK-SD-NEXT: sdot v5.4s, v6.16b, v3.16b
-; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
-; CHECK-SD-NEXT: ld1 { v2.b }[8], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1048
-; CHECK-SD-NEXT: ldr b3, [sp, #80]
-; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8]
-; CHECK-SD-NEXT: add x10, sp, #88
-; CHECK-SD-NEXT: add x8, sp, #536
-; CHECK-SD-NEXT: add x11, sp, #1440
-; CHECK-SD-NEXT: add x9, sp, #56
-; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
-; CHECK-SD-NEXT: ld1 { v2.b }[9], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1056
-; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8]
-; CHECK-SD-NEXT: ld1 { v1.b }[13], [x9]
-; CHECK-SD-NEXT: add x9, sp, #96
-; CHECK-SD-NEXT: add x8, sp, #544
-; CHECK-SD-NEXT: add x10, sp, #1448
-; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v2.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1064
-; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8]
-; CHECK-SD-NEXT: add x10, sp, #104
-; CHECK-SD-NEXT: add x8, sp, #552
-; CHECK-SD-NEXT: add x11, sp, #1456
-; CHECK-SD-NEXT: add x9, sp, #64
-; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10]
-; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1072
-; CHECK-SD-NEXT: ld1 { v7.b }[12], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8]
-; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #112
-; CHECK-SD-NEXT: add x8, sp, #560
-; CHECK-SD-NEXT: add x10, sp, #1464
-; CHECK-SD-NEXT: ld1 { v3.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v2.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1080
-; CHECK-SD-NEXT: ld1 { v7.b }[13], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8]
-; CHECK-SD-NEXT: add x10, sp, #120
-; CHECK-SD-NEXT: add x8, sp, #568
-; CHECK-SD-NEXT: add x11, sp, #1472
-; CHECK-SD-NEXT: add x9, sp, #72
-; CHECK-SD-NEXT: ld1 { v3.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v2.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1088
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v1.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #128
-; CHECK-SD-NEXT: ldr b6, [sp, #1104]
-; CHECK-SD-NEXT: add x10, sp, #1480
-; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
-; CHECK-SD-NEXT: add x8, sp, #1096
-; CHECK-SD-NEXT: add x9, sp, #1112
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9]
-; CHECK-SD-NEXT: add x8, sp, #728
-; CHECK-SD-NEXT: add x9, sp, #576
-; CHECK-SD-NEXT: add x10, sp, #136
-; CHECK-SD-NEXT: ld1 { v17.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1120
-; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9]
-; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #736
-; CHECK-SD-NEXT: ldr b7, [sp, #1232]
-; CHECK-SD-NEXT: ldr b16, [sp, #848]
-; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8]
-; CHECK-SD-NEXT: add x9, sp, #1240
-; CHECK-SD-NEXT: add x10, sp, #856
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1128
-; CHECK-SD-NEXT: add x11, sp, #744
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1248
-; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #864
-; CHECK-SD-NEXT: add x9, sp, #144
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1136
-; CHECK-SD-NEXT: add x12, sp, #752
-; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1256
-; CHECK-SD-NEXT: add x10, sp, #872
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1144
-; CHECK-SD-NEXT: add x11, sp, #760
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1264
-; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #880
-; CHECK-SD-NEXT: add x9, sp, #152
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1152
-; CHECK-SD-NEXT: add x12, sp, #768
-; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1272
-; CHECK-SD-NEXT: add x10, sp, #888
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1160
-; CHECK-SD-NEXT: add x11, sp, #776
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1280
-; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11]
-; CHECK-SD-NEXT: add x11, sp, #896
-; CHECK-SD-NEXT: add x9, sp, #160
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1168
-; CHECK-SD-NEXT: add x12, sp, #784
-; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1288
-; CHECK-SD-NEXT: add x10, sp, #904
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1176
-; CHECK-SD-NEXT: add x11, sp, #792
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1296
-; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11]
-; CHECK-SD-NEXT: add x11, sp, #912
-; CHECK-SD-NEXT: add x9, sp, #168
-; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1184
-; CHECK-SD-NEXT: add x12, sp, #800
-; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1304
-; CHECK-SD-NEXT: add x10, sp, #920
-; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1192
-; CHECK-SD-NEXT: add x11, sp, #808
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1312
-; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11]
-; CHECK-SD-NEXT: add x11, sp, #928
-; CHECK-SD-NEXT: add x9, sp, #176
-; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1200
-; CHECK-SD-NEXT: add x12, sp, #816
-; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1320
-; CHECK-SD-NEXT: add x10, sp, #936
-; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1208
-; CHECK-SD-NEXT: add x11, sp, #824
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8]
-; CHECK-SD-NEXT: add x10, sp, #1328
-; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11]
-; CHECK-SD-NEXT: add x11, sp, #944
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11]
-; CHECK-SD-NEXT: add x8, sp, #1216
-; CHECK-SD-NEXT: add x12, sp, #832
-; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1336
-; CHECK-SD-NEXT: add x10, sp, #952
-; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10]
-; CHECK-SD-NEXT: add x8, sp, #1224
-; CHECK-SD-NEXT: add x11, sp, #840
-; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #192
-; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11]
-; CHECK-SD-NEXT: add x10, sp, #1344
-; CHECK-SD-NEXT: add x11, sp, #960
-; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11]
-; CHECK-SD-NEXT: add x9, sp, #584
-; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b
-; CHECK-SD-NEXT: add x8, sp, #200
-; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b
-; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1352
-; CHECK-SD-NEXT: add x10, sp, #968
-; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9]
-; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10]
-; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b
-; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
-; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
-; CHECK-SD-NEXT: addv s0, v0.4s
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_sdot_v48i8_double:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w11, [sp, #80]
-; CHECK-GI-NEXT: ldr w10, [sp, #208]
-; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: ldr w12, [sp, #344]
-; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
-; CHECK-GI-NEXT: fmov s1, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #336]
-; CHECK-GI-NEXT: fmov s2, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #464]
-; CHECK-GI-NEXT: ldr w9, [sp, #216]
-; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: fmov s3, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #600]
-; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #592]
-; CHECK-GI-NEXT: fmov s4, w10
-; CHECK-GI-NEXT: mov v2.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #472]
-; CHECK-GI-NEXT: ldr w10, [sp, #608]
-; CHECK-GI-NEXT: mov v3.b[1], w12
-; CHECK-GI-NEXT: fmov s5, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: mov v4.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #224]
-; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #352]
-; CHECK-GI-NEXT: ldr w12, [sp, #848]
-; CHECK-GI-NEXT: mov v2.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #480]
-; CHECK-GI-NEXT: mov v5.b[1], w11
-; CHECK-GI-NEXT: mov v3.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #104]
-; CHECK-GI-NEXT: ldr w11, [sp, #16]
-; CHECK-GI-NEXT: mov v4.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #232]
-; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: mov v1.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #360]
-; CHECK-GI-NEXT: fmov s7, w12
-; CHECK-GI-NEXT: mov v2.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #488]
-; CHECK-GI-NEXT: mov v5.b[2], w10
-; CHECK-GI-NEXT: mov v3.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: ldr w10, [sp, #616]
-; CHECK-GI-NEXT: mov v4.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #240]
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #368]
-; CHECK-GI-NEXT: ldr w12, [sp, #1112]
-; CHECK-GI-NEXT: mov v2.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #496]
-; CHECK-GI-NEXT: mov v5.b[3], w10
-; CHECK-GI-NEXT: mov v3.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #120]
-; CHECK-GI-NEXT: ldr w10, [sp, #624]
-; CHECK-GI-NEXT: mov v4.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #248]
-; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v1.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #376]
-; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #504]
-; CHECK-GI-NEXT: mov v5.b[4], w10
-; CHECK-GI-NEXT: mov v3.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: ldr w10, [sp, #632]
-; CHECK-GI-NEXT: mov v4.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #256]
-; CHECK-GI-NEXT: mov v0.b[6], w6
-; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #384]
-; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #512]
-; CHECK-GI-NEXT: mov v5.b[5], w10
-; CHECK-GI-NEXT: mov v3.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #136]
-; CHECK-GI-NEXT: ldr w10, [sp, #640]
-; CHECK-GI-NEXT: mov v4.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #264]
-; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v1.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #392]
-; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #520]
-; CHECK-GI-NEXT: mov v5.b[6], w10
-; CHECK-GI-NEXT: mov v3.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #144]
-; CHECK-GI-NEXT: ldr w10, [sp, #648]
-; CHECK-GI-NEXT: mov v4.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #272]
-; CHECK-GI-NEXT: mov v0.b[8], w11
-; CHECK-GI-NEXT: mov v1.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #400]
-; CHECK-GI-NEXT: ldr w11, [sp, #24]
-; CHECK-GI-NEXT: mov v2.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #528]
-; CHECK-GI-NEXT: mov v5.b[7], w10
-; CHECK-GI-NEXT: mov v3.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #152]
-; CHECK-GI-NEXT: ldr w10, [sp, #656]
-; CHECK-GI-NEXT: mov v4.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #280]
-; CHECK-GI-NEXT: mov v0.b[9], w11
-; CHECK-GI-NEXT: mov v1.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #408]
-; CHECK-GI-NEXT: ldr w11, [sp, #32]
-; CHECK-GI-NEXT: mov v2.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #536]
-; CHECK-GI-NEXT: mov v5.b[8], w10
-; CHECK-GI-NEXT: mov v3.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: ldr w10, [sp, #664]
-; CHECK-GI-NEXT: mov v4.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #288]
-; CHECK-GI-NEXT: mov v0.b[10], w11
-; CHECK-GI-NEXT: mov v1.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #416]
-; CHECK-GI-NEXT: ldr w11, [sp, #40]
-; CHECK-GI-NEXT: mov v2.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #544]
-; CHECK-GI-NEXT: mov v5.b[9], w10
-; CHECK-GI-NEXT: mov v3.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #168]
-; CHECK-GI-NEXT: ldr w10, [sp, #672]
-; CHECK-GI-NEXT: mov v4.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #296]
-; CHECK-GI-NEXT: mov v0.b[11], w11
-; CHECK-GI-NEXT: mov v1.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #424]
-; CHECK-GI-NEXT: ldr w11, [sp, #48]
-; CHECK-GI-NEXT: mov v2.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #552]
-; CHECK-GI-NEXT: mov v5.b[10], w10
-; CHECK-GI-NEXT: mov v3.b[11], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #176]
-; CHECK-GI-NEXT: ldr w10, [sp, #680]
-; CHECK-GI-NEXT: mov v4.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #304]
-; CHECK-GI-NEXT: mov v0.b[12], w11
-; CHECK-GI-NEXT: mov v1.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #432]
-; CHECK-GI-NEXT: ldr w11, [sp, #56]
-; CHECK-GI-NEXT: mov v2.b[12], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #560]
-; CHECK-GI-NEXT: mov v5.b[11], w10
-; CHECK-GI-NEXT: mov v3.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: ldr w10, [sp, #688]
-; CHECK-GI-NEXT: mov v4.b[12], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #312]
-; CHECK-GI-NEXT: mov v0.b[13], w11
-; CHECK-GI-NEXT: mov v1.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #440]
-; CHECK-GI-NEXT: ldr w11, [sp, #64]
-; CHECK-GI-NEXT: mov v2.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #568]
-; CHECK-GI-NEXT: mov v5.b[12], w10
-; CHECK-GI-NEXT: mov v3.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #192]
-; CHECK-GI-NEXT: ldr w10, [sp, #696]
-; CHECK-GI-NEXT: mov v4.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #320]
-; CHECK-GI-NEXT: mov v0.b[14], w11
-; CHECK-GI-NEXT: mov v1.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #448]
-; CHECK-GI-NEXT: ldr w11, [sp, #72]
-; CHECK-GI-NEXT: mov v2.b[14], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #576]
-; CHECK-GI-NEXT: mov v5.b[13], w10
-; CHECK-GI-NEXT: mov v3.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #720]
-; CHECK-GI-NEXT: ldr w10, [sp, #704]
-; CHECK-GI-NEXT: mov v4.b[14], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #728]
-; CHECK-GI-NEXT: mov v0.b[15], w11
-; CHECK-GI-NEXT: fmov s6, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #328]
-; CHECK-GI-NEXT: ldr w11, [sp, #456]
-; CHECK-GI-NEXT: mov v5.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #200]
-; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v2.b[15], w8
-; CHECK-GI-NEXT: mov v3.b[15], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #736]
-; CHECK-GI-NEXT: mov v6.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #584]
-; CHECK-GI-NEXT: ldr w8, [sp, #856]
-; CHECK-GI-NEXT: mov v1.b[15], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #712]
-; CHECK-GI-NEXT: mov v4.b[15], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #976]
-; CHECK-GI-NEXT: mov v7.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1232]
-; CHECK-GI-NEXT: mov v5.b[15], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #984]
-; CHECK-GI-NEXT: mov v6.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1104]
-; CHECK-GI-NEXT: fmov s16, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1360]
-; CHECK-GI-NEXT: fmov s18, w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1368]
-; CHECK-GI-NEXT: fmov s17, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1240]
-; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b
-; CHECK-GI-NEXT: mov v16.b[1], w10
-; CHECK-GI-NEXT: fmov s19, w9
-; CHECK-GI-NEXT: ldr w10, [sp, #864]
-; CHECK-GI-NEXT: mov v18.b[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #992]
-; CHECK-GI-NEXT: ldr w9, [sp, #1120]
-; CHECK-GI-NEXT: mov v17.b[1], w12
-; CHECK-GI-NEXT: mov v7.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1248]
-; CHECK-GI-NEXT: mov v19.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #744]
-; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b
-; CHECK-GI-NEXT: mov v16.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #872]
-; CHECK-GI-NEXT: addv s0, v20.4s
-; CHECK-GI-NEXT: mov v6.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1000]
-; CHECK-GI-NEXT: mov v18.b[2], w10
-; CHECK-GI-NEXT: mov v17.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1376]
-; CHECK-GI-NEXT: ldr w10, [sp, #1128]
-; CHECK-GI-NEXT: mov v7.b[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #880]
-; CHECK-GI-NEXT: addv s1, v21.4s
-; CHECK-GI-NEXT: mov v19.b[2], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #752]
-; CHECK-GI-NEXT: mov v16.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1256]
-; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b
-; CHECK-GI-NEXT: mov v17.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1384]
-; CHECK-GI-NEXT: mov v6.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1008]
-; CHECK-GI-NEXT: mov v18.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1136]
-; CHECK-GI-NEXT: mov v19.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #760]
-; CHECK-GI-NEXT: mov v7.b[4], w11
-; CHECK-GI-NEXT: mov v16.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1264]
-; CHECK-GI-NEXT: ldr w11, [sp, #888]
-; CHECK-GI-NEXT: mov v17.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1392]
-; CHECK-GI-NEXT: mov v6.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1016]
-; CHECK-GI-NEXT: mov v18.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1144]
-; CHECK-GI-NEXT: mov v19.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #768]
-; CHECK-GI-NEXT: mov v7.b[5], w11
-; CHECK-GI-NEXT: mov v16.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1272]
-; CHECK-GI-NEXT: ldr w11, [sp, #896]
-; CHECK-GI-NEXT: mov v17.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1400]
-; CHECK-GI-NEXT: mov v6.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1024]
-; CHECK-GI-NEXT: mov v18.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1152]
-; CHECK-GI-NEXT: mov v19.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #776]
-; CHECK-GI-NEXT: mov v7.b[6], w11
-; CHECK-GI-NEXT: mov v16.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1280]
-; CHECK-GI-NEXT: ldr w11, [sp, #904]
-; CHECK-GI-NEXT: mov v17.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1408]
-; CHECK-GI-NEXT: mov v6.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1032]
-; CHECK-GI-NEXT: mov v18.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1160]
-; CHECK-GI-NEXT: mov v19.b[6], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #784]
-; CHECK-GI-NEXT: mov v7.b[7], w11
-; CHECK-GI-NEXT: mov v16.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1288]
-; CHECK-GI-NEXT: ldr w11, [sp, #912]
-; CHECK-GI-NEXT: mov v17.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1416]
-; CHECK-GI-NEXT: mov v6.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1040]
-; CHECK-GI-NEXT: mov v18.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1168]
-; CHECK-GI-NEXT: mov v19.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #792]
-; CHECK-GI-NEXT: mov v7.b[8], w11
-; CHECK-GI-NEXT: mov v16.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1296]
-; CHECK-GI-NEXT: ldr w11, [sp, #920]
-; CHECK-GI-NEXT: mov v17.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1424]
-; CHECK-GI-NEXT: mov v6.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1048]
-; CHECK-GI-NEXT: mov v18.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1176]
-; CHECK-GI-NEXT: mov v19.b[8], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #800]
-; CHECK-GI-NEXT: mov v7.b[9], w11
-; CHECK-GI-NEXT: mov v16.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1304]
-; CHECK-GI-NEXT: ldr w11, [sp, #928]
-; CHECK-GI-NEXT: mov v17.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1432]
-; CHECK-GI-NEXT: mov v6.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1056]
-; CHECK-GI-NEXT: mov v18.b[9], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1184]
-; CHECK-GI-NEXT: mov v19.b[9], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #808]
-; CHECK-GI-NEXT: mov v7.b[10], w11
-; CHECK-GI-NEXT: mov v16.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1312]
-; CHECK-GI-NEXT: ldr w11, [sp, #936]
-; CHECK-GI-NEXT: mov v17.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1440]
-; CHECK-GI-NEXT: mov v6.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1064]
-; CHECK-GI-NEXT: mov v18.b[10], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1192]
-; CHECK-GI-NEXT: mov v19.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #816]
-; CHECK-GI-NEXT: mov v7.b[11], w11
-; CHECK-GI-NEXT: mov v16.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1320]
-; CHECK-GI-NEXT: ldr w11, [sp, #944]
-; CHECK-GI-NEXT: mov v17.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1448]
-; CHECK-GI-NEXT: mov v6.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1072]
-; CHECK-GI-NEXT: mov v18.b[11], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1200]
-; CHECK-GI-NEXT: mov v19.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #824]
-; CHECK-GI-NEXT: mov v7.b[12], w11
-; CHECK-GI-NEXT: mov v16.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1328]
-; CHECK-GI-NEXT: ldr w11, [sp, #952]
-; CHECK-GI-NEXT: mov v17.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1456]
-; CHECK-GI-NEXT: mov v6.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1080]
-; CHECK-GI-NEXT: mov v18.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1208]
-; CHECK-GI-NEXT: mov v19.b[12], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #832]
-; CHECK-GI-NEXT: mov v7.b[13], w11
-; CHECK-GI-NEXT: mov v16.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1336]
-; CHECK-GI-NEXT: ldr w11, [sp, #960]
-; CHECK-GI-NEXT: mov v17.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1464]
-; CHECK-GI-NEXT: mov v6.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1088]
-; CHECK-GI-NEXT: mov v18.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1216]
-; CHECK-GI-NEXT: mov v19.b[13], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #840]
-; CHECK-GI-NEXT: mov v7.b[14], w11
-; CHECK-GI-NEXT: mov v16.b[14], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1344]
-; CHECK-GI-NEXT: ldr w11, [sp, #968]
-; CHECK-GI-NEXT: mov v17.b[14], w9
-; CHECK-GI-NEXT: mov v6.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1096]
-; CHECK-GI-NEXT: mov v18.b[14], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #1472]
-; CHECK-GI-NEXT: ldr w10, [sp, #1224]
-; CHECK-GI-NEXT: mov v7.b[15], w11
-; CHECK-GI-NEXT: addv s4, v25.4s
-; CHECK-GI-NEXT: mov v16.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #1352]
-; CHECK-GI-NEXT: mov v19.b[14], w9
-; CHECK-GI-NEXT: mov v17.b[15], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #1480]
-; CHECK-GI-NEXT: mov v18.b[15], w8
+; CHECK-NEXT: add x10, sp, #48
+; CHECK-NEXT: ld1 { v4.b }[1], [x12]
+; CHECK-NEXT: add x12, sp, #176
+; CHECK-NEXT: ld1 { v5.b }[2], [x13]
+; CHECK-NEXT: add x13, sp, #680
+; CHECK-NEXT: ld1 { v3.b }[4], [x10]
+; CHECK-NEXT: ld1 { v2.b }[4], [x12]
+; CHECK-NEXT: ld1 { v6.b }[1], [x13]
+; CHECK-NEXT: add x13, sp, #56
+; CHECK-NEXT: ld1 { v0.b }[5], [x9]
+; CHECK-NEXT: mov v1.b[2], w2
+; CHECK-NEXT: add x8, sp, #128
+; CHECK-NEXT: add x14, sp, #184
+; CHECK-NEXT: add x11, sp, #136
+; CHECK-NEXT: ld1 { v3.b }[5], [x13]
+; CHECK-NEXT: add x13, sp, #552
+; CHECK-NEXT: ld1 { v2.b }[5], [x14]
+; CHECK-NEXT: ld1 { v16.b }[1], [x13]
+; CHECK-NEXT: add x14, sp, #624
+; CHECK-NEXT: ld1 { v0.b }[6], [x8]
+; CHECK-NEXT: add x8, sp, #688
+; CHECK-NEXT: add x13, sp, #504
+; CHECK-NEXT: ld1 { v4.b }[2], [x14]
+; CHECK-NEXT: ld1 { v6.b }[2], [x8]
+; CHECK-NEXT: add x8, sp, #560
+; CHECK-NEXT: ld1 { v5.b }[3], [x13]
+; CHECK-NEXT: ld1 { v16.b }[2], [x8]
+; CHECK-NEXT: mov v1.b[3], w3
+; CHECK-NEXT: add x9, sp, #64
+; CHECK-NEXT: add x15, sp, #632
+; CHECK-NEXT: ld1 { v3.b }[6], [x9]
+; CHECK-NEXT: ld1 { v0.b }[7], [x11]
+; CHECK-NEXT: ld1 { v4.b }[3], [x15]
+; CHECK-NEXT: add x8, sp, #696
+; CHECK-NEXT: add x9, sp, #568
+; CHECK-NEXT: add x11, sp, #512
+; CHECK-NEXT: ld1 { v6.b }[3], [x8]
+; CHECK-NEXT: ld1 { v16.b }[3], [x9]
+; CHECK-NEXT: ld1 { v5.b }[4], [x11]
+; CHECK-NEXT: add x8, sp, #640
+; CHECK-NEXT: mov v1.b[4], w4
+; CHECK-NEXT: ld1 { v4.b }[4], [x8]
+; CHECK-NEXT: add x8, sp, #704
+; CHECK-NEXT: add x9, sp, #576
+; CHECK-NEXT: add x11, sp, #520
+; CHECK-NEXT: ld1 { v6.b }[4], [x8]
+; CHECK-NEXT: ld1 { v16.b }[4], [x9]
+; CHECK-NEXT: ld1 { v5.b }[5], [x11]
+; CHECK-NEXT: ldr b18, [sp, #736]
+; CHECK-NEXT: add x12, sp, #192
+; CHECK-NEXT: ld1 { v2.b }[6], [x12]
+; CHECK-NEXT: add x8, sp, #648
+; CHECK-NEXT: add x9, sp, #528
+; CHECK-NEXT: add x11, sp, #712
+; CHECK-NEXT: add x12, sp, #584
+; CHECK-NEXT: sshll v18.8h, v18.8b, #0
+; CHECK-NEXT: mov v1.b[5], w5
+; CHECK-NEXT: ld1 { v6.b }[5], [x11]
+; CHECK-NEXT: ld1 { v16.b }[5], [x12]
+; CHECK-NEXT: ld1 { v4.b }[5], [x8]
+; CHECK-NEXT: ld1 { v5.b }[6], [x9]
+; CHECK-NEXT: movi v17.2d, #0000000000000000
+; CHECK-NEXT: add x8, sp, #656
+; CHECK-NEXT: add x9, sp, #536
+; CHECK-NEXT: add x11, sp, #720
+; CHECK-NEXT: add x12, sp, #592
+; CHECK-NEXT: sshll v18.4s, v18.4h, #0
+; CHECK-NEXT: ldr b7, [sp, #208]
+; CHECK-NEXT: ld1 { v6.b }[6], [x11]
+; CHECK-NEXT: ld1 { v16.b }[6], [x12]
+; CHECK-NEXT: ld1 { v4.b }[6], [x8]
+; CHECK-NEXT: ld1 { v5.b }[7], [x9]
+; CHECK-NEXT: mov v1.b[6], w6
+; CHECK-NEXT: sshll v7.8h, v7.8b, #0
+; CHECK-NEXT: add x8, sp, #664
+; CHECK-NEXT: add x9, sp, #728
+; CHECK-NEXT: add x11, sp, #600
+; CHECK-NEXT: mov v17.s[0], v18.s[0]
+; CHECK-NEXT: ld1 { v6.b }[7], [x9]
+; CHECK-NEXT: ld1 { v16.b }[7], [x11]
+; CHECK-NEXT: ld1 { v4.b }[7], [x8]
+; CHECK-NEXT: sshll v5.8h, v5.8b, #0
+; CHECK-NEXT: movi v18.2d, #0000000000000000
+; CHECK-NEXT: add x10, sp, #200
+; CHECK-NEXT: mov v1.b[7], w7
+; CHECK-NEXT: add x9, sp, #72
+; CHECK-NEXT: sshll v7.4s, v7.4h, #0
+; CHECK-NEXT: ld1 { v2.b }[7], [x10]
+; CHECK-NEXT: ld1 { v3.b }[7], [x9]
+; CHECK-NEXT: sshll v6.8h, v6.8b, #0
+; CHECK-NEXT: sshll v16.8h, v16.8b, #0
+; CHECK-NEXT: sshll v4.8h, v4.8b, #0
+; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: mov v18.s[0], v7.s[0]
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: sshll v2.8h, v2.8b, #0
+; CHECK-NEXT: sshll v3.8h, v3.8b, #0
+; CHECK-NEXT: saddl2 v7.4s, v16.8h, v6.8h
+; CHECK-NEXT: saddl2 v5.4s, v5.8h, v4.8h
+; CHECK-NEXT: saddl v6.4s, v16.4h, v6.4h
+; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h
+; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h
+; CHECK-NEXT: saddl2 v16.4s, v3.8h, v2.8h
+; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h
+; CHECK-NEXT: add v5.4s, v5.4s, v7.4s
+; CHECK-NEXT: add v4.4s, v4.4s, v6.4s
+; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h
+; CHECK-NEXT: add v6.4s, v17.4s, v16.4s
+; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
+; CHECK-NEXT: add v1.4s, v4.4s, v5.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: add v1.4s, v6.4s, v1.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+entry:
+ %az = sext <33 x i8> %a to <33 x i32>
+ %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)
+ %cz = sext <33 x i8> %c to <33 x i32>
+ %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz)
+ %x = add i32 %r1, %r2
+ ret i32 %x
+}
+
+define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_udot_v48i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q1, [x0, #32]
+; CHECK-SD-NEXT: ldr q2, [x1, #32]
+; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT: ldp q3, q1, [x0]
+; CHECK-SD-NEXT: ldp q4, q2, [x1]
+; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_udot_v48i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr q7, [x0, #32]
+; CHECK-GI-NEXT: ldp q3, q4, [x0]
+; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT: ldp q5, q6, [x1]
+; CHECK-GI-NEXT: ldr q16, [x1, #32]
+; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b
+; CHECK-GI-NEXT: udot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: addv s2, v2.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <48 x i8>, ptr %a
+ %1 = zext <48 x i8> %0 to <48 x i32>
+ %2 = load <48 x i8>, ptr %b
+ %3 = zext <48 x i8> %2 to <48 x i32>
+ %4 = mul nuw nsw <48 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
+ %op.extra = add i32 %5, %sum
+ ret i32 %op.extra
+}
+
+define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
+; CHECK-SD-LABEL: test_udot_v48i8_nomla:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.16b, #1
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q2, [x0, #32]
+; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT: ldp q3, q2, [x0]
+; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b
+; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b
+; CHECK-SD-NEXT: addv s0, v1.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_udot_v48i8_nomla:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v0.16b, #1
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr q6, [x0, #32]
+; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT: ldp q4, q5, [x0]
+; CHECK-GI-NEXT: movi v3.2d, #0000000000000000
+; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b
+; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b
+; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b
+; CHECK-GI-NEXT: addv s0, v1.4s
+; CHECK-GI-NEXT: addv s1, v2.4s
+; CHECK-GI-NEXT: addv s2, v3.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w9, s1
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: add w0, w8, w9
+; CHECK-GI-NEXT: ret
+entry:
+ %0 = load <48 x i8>, ptr %a1
+ %1 = zext <48 x i8> %0 to <48 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1)
+ ret i32 %2
+}
+define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
+; CHECK-SD-LABEL: test_sdot_v48i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
+; CHECK-SD-NEXT: ldr q1, [x0, #32]
+; CHECK-SD-NEXT: ldr q2, [x1, #32]
+; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT: ldp q3, q1, [x0]
+; CHECK-SD-NEXT: ldp q4, q2, [x1]
+; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b
+; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_sdot_v48i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr q7, [x0, #32]
+; CHECK-GI-NEXT: ldp q3, q4, [x0]
+; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT: ldp q5, q6, [x1]
+; CHECK-GI-NEXT: ldr q16, [x1, #32]
+; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b
+; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b
+; CHECK-GI-NEXT: sdot v2.4s, v16.16b, v7.16b
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: addv s1, v1.4s
+; CHECK-GI-NEXT: addv s2, v2.4s
; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w11, s4
-; CHECK-GI-NEXT: mov v19.b[15], w9
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b
-; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
-; CHECK-GI-NEXT: add w8, w8, w11
-; CHECK-GI-NEXT: addv s2, v22.4s
-; CHECK-GI-NEXT: addv s3, v23.4s
-; CHECK-GI-NEXT: addv s5, v24.4s
; CHECK-GI-NEXT: fmov w9, s2
-; CHECK-GI-NEXT: fmov w10, s3
-; CHECK-GI-NEXT: add w9, w9, w10
-; CHECK-GI-NEXT: fmov w10, s5
-; CHECK-GI-NEXT: add w9, w9, w10
-; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: add w0, w8, w2
; CHECK-GI-NEXT: ret
entry:
- %az = sext <48 x i8> %a to <48 x i32>
- %bz = sext <48 x i8> %b to <48 x i32>
- %m1 = mul nuw nsw <48 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
- %cz = sext <48 x i8> %c to <48 x i32>
- %dz = sext <48 x i8> %d to <48 x i32>
- %m2 = mul nuw nsw <48 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
- %x = add i32 %r1, %r2
- ret i32 %x
+ %0 = load <48 x i8>, ptr %a
+ %1 = sext <48 x i8> %0 to <48 x i32>
+ %2 = load <48 x i8>, ptr %b
+ %3 = sext <48 x i8> %2 to <48 x i32>
+ %4 = mul nsw <48 x i32> %3, %1
+ %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
+ %op.extra = add nsw i32 %5, %sum
+ ret i32 %op.extra
}
-define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
+define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v48i8_double:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: .cfi_offset w29, -16
-; CHECK-SD-NEXT: ldr b5, [sp, #208]
+; CHECK-SD-NEXT: ldr b3, [sp, #592]
+; CHECK-SD-NEXT: add x8, sp, #600
+; CHECK-SD-NEXT: ldr b6, [sp, #208]
+; CHECK-SD-NEXT: ldr b0, [sp, #336]
+; CHECK-SD-NEXT: add x9, sp, #344
+; CHECK-SD-NEXT: ldr b2, [sp, #464]
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #216
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: ldr b4, [sp, #976]
-; CHECK-SD-NEXT: add x9, sp, #984
-; CHECK-SD-NEXT: add x12, sp, #328
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
+; CHECK-SD-NEXT: add x10, sp, #624
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #608
+; CHECK-SD-NEXT: ld1 { v0.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #232
+; CHECK-SD-NEXT: fmov s1, w0
+; CHECK-SD-NEXT: ldr b7, [sp, #1360]
+; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #224
-; CHECK-SD-NEXT: movi v1.16b, #1
-; CHECK-SD-NEXT: mov v0.b[1], w1
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9]
-; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT: add x11, sp, #992
-; CHECK-SD-NEXT: ldr b6, [sp, #720]
-; CHECK-SD-NEXT: ldr b7, [sp, #80]
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #232
-; CHECK-SD-NEXT: add x13, sp, #88
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
-; CHECK-SD-NEXT: add x13, sp, #856
-; CHECK-SD-NEXT: mov v0.b[2], w2
-; CHECK-SD-NEXT: add x14, sp, #1008
-; CHECK-SD-NEXT: add x15, sp, #872
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #240
-; CHECK-SD-NEXT: add x16, sp, #888
-; CHECK-SD-NEXT: add x10, sp, #16
-; CHECK-SD-NEXT: add x9, sp, #24
-; CHECK-SD-NEXT: add x11, sp, #40
-; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #248
-; CHECK-SD-NEXT: mov v0.b[3], w3
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #256
-; CHECK-SD-NEXT: mov v0.b[4], w4
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
-; CHECK-SD-NEXT: add x8, sp, #264
-; CHECK-SD-NEXT: mov v0.b[5], w5
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #272
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8]
-; CHECK-SD-NEXT: add x8, sp, #280
-; CHECK-SD-NEXT: mov v0.b[6], w6
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8]
-; CHECK-SD-NEXT: add x8, sp, #288
-; CHECK-SD-NEXT: mov v0.b[7], w7
-; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #296
-; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10]
-; CHECK-SD-NEXT: add x10, sp, #128
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #304
-; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9]
-; CHECK-SD-NEXT: add x9, sp, #136
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #312
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #320
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT: add x8, sp, #32
-; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #144
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12]
-; CHECK-SD-NEXT: add x12, sp, #728
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1000
-; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12]
-; CHECK-SD-NEXT: add x12, sp, #736
-; CHECK-SD-NEXT: add x11, sp, #920
-; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b
-; CHECK-SD-NEXT: ldr b5, [sp, #848]
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
-; CHECK-SD-NEXT: add x12, sp, #48
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13]
-; CHECK-SD-NEXT: add x13, sp, #744
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14]
-; CHECK-SD-NEXT: add x14, sp, #96
-; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13]
-; CHECK-SD-NEXT: add x13, sp, #864
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14]
-; CHECK-SD-NEXT: add x14, sp, #1016
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT: add x13, sp, #752
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14]
-; CHECK-SD-NEXT: add x14, sp, #104
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
-; CHECK-SD-NEXT: add x13, sp, #1024
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14]
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15]
-; CHECK-SD-NEXT: add x15, sp, #760
-; CHECK-SD-NEXT: add x14, sp, #112
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
-; CHECK-SD-NEXT: add x13, sp, #880
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15]
-; CHECK-SD-NEXT: add x15, sp, #1032
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
-; CHECK-SD-NEXT: add x14, sp, #768
-; CHECK-SD-NEXT: add x13, sp, #120
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15]
-; CHECK-SD-NEXT: add x15, sp, #1040
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14]
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
-; CHECK-SD-NEXT: add x13, sp, #776
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16]
-; CHECK-SD-NEXT: add x14, sp, #1048
-; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15]
-; CHECK-SD-NEXT: add x15, sp, #896
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13]
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT: add x10, sp, #784
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15]
-; CHECK-SD-NEXT: add x13, sp, #1056
-; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
-; CHECK-SD-NEXT: add x14, sp, #904
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #792
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14]
-; CHECK-SD-NEXT: add x10, sp, #1064
-; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13]
-; CHECK-SD-NEXT: add x13, sp, #912
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
+; CHECK-SD-NEXT: add x11, sp, #648
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #616
+; CHECK-SD-NEXT: add x12, sp, #376
+; CHECK-SD-NEXT: mov v1.b[1], w1
+; CHECK-SD-NEXT: ldr b16, [sp, #976]
+; CHECK-SD-NEXT: add x14, sp, #288
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #632
+; CHECK-SD-NEXT: add x15, sp, #408
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #472
+; CHECK-SD-NEXT: add x13, sp, #696
+; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT: add x9, sp, #240
+; CHECK-SD-NEXT: add x16, sp, #448
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #352
+; CHECK-SD-NEXT: mov v1.b[2], w2
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v0.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1368
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10]
+; CHECK-SD-NEXT: add x10, sp, #248
+; CHECK-SD-NEXT: add x9, sp, #640
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8]
+; CHECK-SD-NEXT: add x8, sp, #656
+; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
+; CHECK-SD-NEXT: add x10, sp, #360
+; CHECK-SD-NEXT: mov v1.b[3], w3
+; CHECK-SD-NEXT: ld1 { v0.b }[3], [x10]
+; CHECK-SD-NEXT: add x10, sp, #256
+; CHECK-SD-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
+; CHECK-SD-NEXT: add x9, sp, #368
+; CHECK-SD-NEXT: ldr b17, [sp, #720]
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #984
+; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
+; CHECK-SD-NEXT: add x10, sp, #664
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11]
+; CHECK-SD-NEXT: add x11, sp, #264
+; CHECK-SD-NEXT: mov v1.b[4], w4
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11]
+; CHECK-SD-NEXT: add x9, sp, #672
+; CHECK-SD-NEXT: add x11, sp, #680
+; CHECK-SD-NEXT: ld1 { v0.b }[5], [x12]
+; CHECK-SD-NEXT: add x12, sp, #480
+; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12]
+; CHECK-SD-NEXT: add x12, sp, #272
+; CHECK-SD-NEXT: ld1 { v3.b }[8], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x12]
+; CHECK-SD-NEXT: add x12, sp, #384
+; CHECK-SD-NEXT: mov v1.b[5], w5
+; CHECK-SD-NEXT: ld1 { v0.b }[6], [x12]
+; CHECK-SD-NEXT: add x12, sp, #280
+; CHECK-SD-NEXT: add x8, sp, #688
+; CHECK-SD-NEXT: ld1 { v3.b }[9], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1376
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT: add x10, sp, #392
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12]
+; CHECK-SD-NEXT: ld1 { v0.b }[7], [x10]
+; CHECK-SD-NEXT: mov v1.b[6], w6
+; CHECK-SD-NEXT: add x12, sp, #704
+; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
+; CHECK-SD-NEXT: add x9, sp, #400
+; CHECK-SD-NEXT: add x10, sp, #712
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x14]
+; CHECK-SD-NEXT: add x14, sp, #992
+; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[2], [x14]
+; CHECK-SD-NEXT: add x14, sp, #296
+; CHECK-SD-NEXT: ld1 { v3.b }[11], [x11]
+; CHECK-SD-NEXT: add x9, sp, #304
+; CHECK-SD-NEXT: add x11, sp, #312
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x14]
+; CHECK-SD-NEXT: mov v1.b[7], w7
+; CHECK-SD-NEXT: add x14, sp, #320
+; CHECK-SD-NEXT: ld1 { v0.b }[9], [x15]
+; CHECK-SD-NEXT: add x15, sp, #328
+; CHECK-SD-NEXT: ld1 { v3.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #416
+; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1384
+; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: add x9, sp, #424
+; CHECK-SD-NEXT: ld1 { v3.b }[13], [x13]
+; CHECK-SD-NEXT: add x8, sp, #432
+; CHECK-SD-NEXT: add x13, sp, #440
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11]
+; CHECK-SD-NEXT: add x11, sp, #16
+; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1000
+; CHECK-SD-NEXT: ld1 { v1.b }[8], [x11]
+; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v3.b }[14], [x12]
+; CHECK-SD-NEXT: add x12, sp, #488
+; CHECK-SD-NEXT: ld1 { v6.b }[14], [x14]
+; CHECK-SD-NEXT: add x14, sp, #1392
+; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12]
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
+; CHECK-SD-NEXT: add x11, sp, #1008
+; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8]
+; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1400
+; CHECK-SD-NEXT: ld1 { v3.b }[15], [x10]
+; CHECK-SD-NEXT: add x10, sp, #496
+; CHECK-SD-NEXT: add x9, sp, #24
+; CHECK-SD-NEXT: ld1 { v6.b }[15], [x15]
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8]
+; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1016
+; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13]
+; CHECK-SD-NEXT: add x8, sp, #1408
+; CHECK-SD-NEXT: ld1 { v1.b }[9], [x9]
+; CHECK-SD-NEXT: add x9, sp, #504
+; CHECK-SD-NEXT: add x10, sp, #512
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1024
+; CHECK-SD-NEXT: add x8, sp, #32
+; CHECK-SD-NEXT: ld1 { v16.b }[6], [x9]
+; CHECK-SD-NEXT: ld1 { v0.b }[14], [x16]
+; CHECK-SD-NEXT: ld1 { v1.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1416
+; CHECK-SD-NEXT: add x9, sp, #456
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8]
+; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1032
+; CHECK-SD-NEXT: add x8, sp, #40
+; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v0.b }[15], [x9]
+; CHECK-SD-NEXT: ld1 { v1.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1424
+; CHECK-SD-NEXT: add x9, sp, #520
; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
-; CHECK-SD-NEXT: add x9, sp, #800
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13]
-; CHECK-SD-NEXT: add x8, sp, #152
-; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1072
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9]
+; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1040
+; CHECK-SD-NEXT: add x8, sp, #48
+; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9]
+; CHECK-SD-NEXT: add x10, sp, #528
+; CHECK-SD-NEXT: ld1 { v1.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1432
+; CHECK-SD-NEXT: sdot v5.4s, v6.16b, v3.16b
; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
-; CHECK-SD-NEXT: add x9, sp, #808
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
-; CHECK-SD-NEXT: add x8, sp, #56
-; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10]
-; CHECK-SD-NEXT: add x10, sp, #160
-; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9]
-; CHECK-SD-NEXT: add x9, sp, #928
-; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1080
-; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
-; CHECK-SD-NEXT: add x8, sp, #816
-; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
+; CHECK-SD-NEXT: ld1 { v2.b }[8], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1048
+; CHECK-SD-NEXT: ldr b3, [sp, #80]
+; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8]
+; CHECK-SD-NEXT: add x10, sp, #88
+; CHECK-SD-NEXT: add x8, sp, #536
+; CHECK-SD-NEXT: add x11, sp, #1440
+; CHECK-SD-NEXT: add x9, sp, #56
+; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10]
+; CHECK-SD-NEXT: ld1 { v2.b }[9], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1056
+; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11]
+; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[13], [x9]
+; CHECK-SD-NEXT: add x9, sp, #96
+; CHECK-SD-NEXT: add x8, sp, #544
+; CHECK-SD-NEXT: add x10, sp, #1448
+; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9]
+; CHECK-SD-NEXT: ld1 { v2.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1064
+; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8]
+; CHECK-SD-NEXT: add x10, sp, #104
+; CHECK-SD-NEXT: add x8, sp, #552
+; CHECK-SD-NEXT: add x11, sp, #1456
+; CHECK-SD-NEXT: add x9, sp, #64
+; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10]
+; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1072
+; CHECK-SD-NEXT: ld1 { v7.b }[12], [x11]
+; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
+; CHECK-SD-NEXT: add x9, sp, #112
+; CHECK-SD-NEXT: add x8, sp, #560
+; CHECK-SD-NEXT: add x10, sp, #1464
+; CHECK-SD-NEXT: ld1 { v3.b }[4], [x9]
+; CHECK-SD-NEXT: ld1 { v2.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1080
+; CHECK-SD-NEXT: ld1 { v7.b }[13], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8]
+; CHECK-SD-NEXT: add x10, sp, #120
+; CHECK-SD-NEXT: add x8, sp, #568
+; CHECK-SD-NEXT: add x11, sp, #1472
+; CHECK-SD-NEXT: add x9, sp, #72
+; CHECK-SD-NEXT: ld1 { v3.b }[5], [x10]
+; CHECK-SD-NEXT: ld1 { v2.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1088
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11]
+; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v1.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #128
+; CHECK-SD-NEXT: ldr b6, [sp, #1104]
+; CHECK-SD-NEXT: add x10, sp, #1480
+; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9]
+; CHECK-SD-NEXT: add x8, sp, #1096
+; CHECK-SD-NEXT: add x9, sp, #1112
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT: add x8, sp, #728
+; CHECK-SD-NEXT: add x9, sp, #576
+; CHECK-SD-NEXT: add x10, sp, #136
+; CHECK-SD-NEXT: ld1 { v17.b }[1], [x8]
+; CHECK-SD-NEXT: add x8, sp, #1120
+; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9]
+; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #736
+; CHECK-SD-NEXT: ldr b7, [sp, #1232]
+; CHECK-SD-NEXT: ldr b16, [sp, #848]
+; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10]
+; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8]
+; CHECK-SD-NEXT: add x9, sp, #1240
+; CHECK-SD-NEXT: add x10, sp, #856
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1128
+; CHECK-SD-NEXT: add x11, sp, #744
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1248
+; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11]
+; CHECK-SD-NEXT: add x11, sp, #864
+; CHECK-SD-NEXT: add x9, sp, #144
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1136
+; CHECK-SD-NEXT: add x12, sp, #752
+; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1256
+; CHECK-SD-NEXT: add x10, sp, #872
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1144
+; CHECK-SD-NEXT: add x11, sp, #760
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1264
+; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11]
+; CHECK-SD-NEXT: add x11, sp, #880
+; CHECK-SD-NEXT: add x9, sp, #152
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1152
+; CHECK-SD-NEXT: add x12, sp, #768
+; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1272
+; CHECK-SD-NEXT: add x10, sp, #888
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1160
+; CHECK-SD-NEXT: add x11, sp, #776
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1280
+; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11]
+; CHECK-SD-NEXT: add x11, sp, #896
+; CHECK-SD-NEXT: add x9, sp, #160
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1168
+; CHECK-SD-NEXT: add x12, sp, #784
+; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1288
+; CHECK-SD-NEXT: add x10, sp, #904
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1176
+; CHECK-SD-NEXT: add x11, sp, #792
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1296
+; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11]
+; CHECK-SD-NEXT: add x11, sp, #912
; CHECK-SD-NEXT: add x9, sp, #168
-; CHECK-SD-NEXT: add x10, sp, #176
+; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1184
+; CHECK-SD-NEXT: add x12, sp, #800
+; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9]
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8]
+; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1304
+; CHECK-SD-NEXT: add x10, sp, #920
+; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1192
+; CHECK-SD-NEXT: add x11, sp, #808
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1312
+; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11]
+; CHECK-SD-NEXT: add x11, sp, #928
+; CHECK-SD-NEXT: add x9, sp, #176
+; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1200
+; CHECK-SD-NEXT: add x12, sp, #816
+; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9]
; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #936
-; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1088
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #64
-; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #824
-; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
-; CHECK-SD-NEXT: add x9, sp, #944
-; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1096
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
-; CHECK-SD-NEXT: add x8, sp, #832
-; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10]
+; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1320
+; CHECK-SD-NEXT: add x10, sp, #936
+; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1208
+; CHECK-SD-NEXT: add x11, sp, #824
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8]
+; CHECK-SD-NEXT: add x10, sp, #1328
+; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11]
+; CHECK-SD-NEXT: add x11, sp, #944
; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: add x10, sp, #72
+; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11]
+; CHECK-SD-NEXT: add x8, sp, #1216
+; CHECK-SD-NEXT: add x12, sp, #832
+; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9]
; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
-; CHECK-SD-NEXT: add x8, sp, #952
+; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12]
+; CHECK-SD-NEXT: add x9, sp, #1336
+; CHECK-SD-NEXT: add x10, sp, #952
; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #840
-; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
-; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b
-; CHECK-SD-NEXT: add x9, sp, #192
+; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10]
+; CHECK-SD-NEXT: add x8, sp, #1224
+; CHECK-SD-NEXT: add x11, sp, #840
; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #960
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT: add x8, sp, #192
+; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11]
+; CHECK-SD-NEXT: add x10, sp, #1344
+; CHECK-SD-NEXT: add x11, sp, #960
+; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10]
+; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11]
+; CHECK-SD-NEXT: add x9, sp, #584
+; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b
; CHECK-SD-NEXT: add x8, sp, #200
-; CHECK-SD-NEXT: add x9, sp, #968
-; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
-; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b
-; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b
-; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b
+; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1352
+; CHECK-SD-NEXT: add x10, sp, #968
+; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9]
+; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10]
+; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b
+; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b
+; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT: ret
;
-; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
+; CHECK-GI-LABEL: test_sdot_v48i8_double:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w10, [sp, #80]
-; CHECK-GI-NEXT: ldr w11, [sp, #208]
+; CHECK-GI-NEXT: ldr w11, [sp, #80]
+; CHECK-GI-NEXT: ldr w10, [sp, #208]
; CHECK-GI-NEXT: fmov s0, w0
-; CHECK-GI-NEXT: ldr w9, [sp, #88]
-; CHECK-GI-NEXT: ldr w12, [sp, #728]
-; CHECK-GI-NEXT: movi v6.16b, #1
-; CHECK-GI-NEXT: fmov s1, w10
-; CHECK-GI-NEXT: fmov s2, w11
-; CHECK-GI-NEXT: ldr w11, [sp, #720]
-; CHECK-GI-NEXT: ldr w10, [sp, #216]
+; CHECK-GI-NEXT: ldr w8, [sp, #88]
+; CHECK-GI-NEXT: ldr w12, [sp, #344]
+; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: fmov s1, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #336]
+; CHECK-GI-NEXT: fmov s2, w10
+; CHECK-GI-NEXT: ldr w10, [sp, #464]
+; CHECK-GI-NEXT: ldr w9, [sp, #216]
; CHECK-GI-NEXT: mov v0.b[1], w1
-; CHECK-GI-NEXT: ldr w13, [sp, #856]
; CHECK-GI-NEXT: fmov s3, w11
-; CHECK-GI-NEXT: ldr w8, [sp, #96]
-; CHECK-GI-NEXT: ldr w11, [sp, #224]
-; CHECK-GI-NEXT: mov v1.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #848]
-; CHECK-GI-NEXT: mov v2.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #976]
-; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
-; CHECK-GI-NEXT: fmov s4, w9
+; CHECK-GI-NEXT: ldr w11, [sp, #600]
+; CHECK-GI-NEXT: movi v21.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #592]
+; CHECK-GI-NEXT: fmov s4, w10
+; CHECK-GI-NEXT: mov v2.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #472]
+; CHECK-GI-NEXT: ldr w10, [sp, #608]
; CHECK-GI-NEXT: mov v3.b[1], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #984]
-; CHECK-GI-NEXT: fmov s5, w10
+; CHECK-GI-NEXT: fmov s5, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #96]
+; CHECK-GI-NEXT: mov v4.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #224]
; CHECK-GI-NEXT: mov v0.b[2], w2
-; CHECK-GI-NEXT: ldr w10, [sp, #736]
; CHECK-GI-NEXT: mov v1.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #864]
-; CHECK-GI-NEXT: mov v2.b[2], w11
-; CHECK-GI-NEXT: mov v4.b[1], w13
-; CHECK-GI-NEXT: ldr w11, [sp, #992]
-; CHECK-GI-NEXT: ldr w12, [sp, #776]
-; CHECK-GI-NEXT: mov v5.b[1], w9
-; CHECK-GI-NEXT: mov v3.b[2], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #104]
-; CHECK-GI-NEXT: ldr w10, [sp, #232]
+; CHECK-GI-NEXT: ldr w8, [sp, #352]
+; CHECK-GI-NEXT: ldr w12, [sp, #848]
+; CHECK-GI-NEXT: mov v2.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #480]
+; CHECK-GI-NEXT: mov v5.b[1], w11
+; CHECK-GI-NEXT: mov v3.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #104]
+; CHECK-GI-NEXT: ldr w11, [sp, #16]
+; CHECK-GI-NEXT: mov v4.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #232]
; CHECK-GI-NEXT: mov v0.b[3], w3
-; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v1.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #872]
-; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
-; CHECK-GI-NEXT: mov v4.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #744]
-; CHECK-GI-NEXT: mov v2.b[3], w10
-; CHECK-GI-NEXT: mov v5.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1000]
-; CHECK-GI-NEXT: ldr w10, [sp, #240]
+; CHECK-GI-NEXT: mov v1.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #360]
+; CHECK-GI-NEXT: fmov s7, w12
+; CHECK-GI-NEXT: mov v2.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #488]
+; CHECK-GI-NEXT: mov v5.b[2], w10
; CHECK-GI-NEXT: mov v3.b[3], w8
; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: mov v0.b[4], w4
-; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
-; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: ldr w10, [sp, #616]
; CHECK-GI-NEXT: mov v4.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #752]
+; CHECK-GI-NEXT: ldr w9, [sp, #240]
+; CHECK-GI-NEXT: mov v0.b[4], w4
; CHECK-GI-NEXT: mov v1.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #880]
-; CHECK-GI-NEXT: mov v5.b[3], w11
-; CHECK-GI-NEXT: mov v2.b[4], w10
-; CHECK-GI-NEXT: mov v3.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #120]
-; CHECK-GI-NEXT: ldr w11, [sp, #1008]
-; CHECK-GI-NEXT: ldr w10, [sp, #248]
+; CHECK-GI-NEXT: ldr w8, [sp, #368]
+; CHECK-GI-NEXT: ldr w12, [sp, #1112]
+; CHECK-GI-NEXT: mov v2.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #496]
+; CHECK-GI-NEXT: mov v5.b[3], w10
+; CHECK-GI-NEXT: mov v3.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #120]
+; CHECK-GI-NEXT: ldr w10, [sp, #624]
+; CHECK-GI-NEXT: mov v4.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #248]
; CHECK-GI-NEXT: mov v0.b[5], w5
-; CHECK-GI-NEXT: mov v4.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #760]
-; CHECK-GI-NEXT: mov v1.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #888]
-; CHECK-GI-NEXT: mov v5.b[4], w11
-; CHECK-GI-NEXT: mov v2.b[5], w10
+; CHECK-GI-NEXT: mov v1.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #376]
+; CHECK-GI-NEXT: movi v22.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #504]
+; CHECK-GI-NEXT: mov v5.b[4], w10
; CHECK-GI-NEXT: mov v3.b[5], w8
; CHECK-GI-NEXT: ldr w8, [sp, #128]
-; CHECK-GI-NEXT: ldr w11, [sp, #1016]
-; CHECK-GI-NEXT: ldr w10, [sp, #256]
-; CHECK-GI-NEXT: mov v0.b[6], w6
+; CHECK-GI-NEXT: ldr w10, [sp, #632]
; CHECK-GI-NEXT: mov v4.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #768]
+; CHECK-GI-NEXT: ldr w9, [sp, #256]
+; CHECK-GI-NEXT: mov v0.b[6], w6
; CHECK-GI-NEXT: mov v1.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #896]
-; CHECK-GI-NEXT: mov v5.b[5], w11
-; CHECK-GI-NEXT: mov v2.b[6], w10
-; CHECK-GI-NEXT: mov v3.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #136]
-; CHECK-GI-NEXT: ldr w11, [sp, #1024]
-; CHECK-GI-NEXT: ldr w10, [sp, #264]
+; CHECK-GI-NEXT: ldr w8, [sp, #384]
+; CHECK-GI-NEXT: movi v23.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #512]
+; CHECK-GI-NEXT: mov v5.b[5], w10
+; CHECK-GI-NEXT: mov v3.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #136]
+; CHECK-GI-NEXT: ldr w10, [sp, #640]
+; CHECK-GI-NEXT: mov v4.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #264]
; CHECK-GI-NEXT: mov v0.b[7], w7
-; CHECK-GI-NEXT: mov v4.b[6], w8
-; CHECK-GI-NEXT: mov v1.b[7], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #904]
-; CHECK-GI-NEXT: mov v5.b[6], w11
-; CHECK-GI-NEXT: mov v2.b[7], w10
-; CHECK-GI-NEXT: ldr w8, [sp, #16]
-; CHECK-GI-NEXT: mov v3.b[7], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #144]
-; CHECK-GI-NEXT: ldr w12, [sp, #1032]
-; CHECK-GI-NEXT: mov v0.b[8], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #784]
-; CHECK-GI-NEXT: ldr w11, [sp, #272]
+; CHECK-GI-NEXT: mov v1.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #392]
+; CHECK-GI-NEXT: movi v24.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #520]
+; CHECK-GI-NEXT: mov v5.b[6], w10
+; CHECK-GI-NEXT: mov v3.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #144]
+; CHECK-GI-NEXT: ldr w10, [sp, #648]
; CHECK-GI-NEXT: mov v4.b[7], w9
-; CHECK-GI-NEXT: mov v1.b[8], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #912]
-; CHECK-GI-NEXT: mov v5.b[7], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #24]
-; CHECK-GI-NEXT: ldr w12, [sp, #1040]
+; CHECK-GI-NEXT: ldr w9, [sp, #272]
+; CHECK-GI-NEXT: mov v0.b[8], w11
+; CHECK-GI-NEXT: mov v1.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #400]
+; CHECK-GI-NEXT: ldr w11, [sp, #24]
+; CHECK-GI-NEXT: mov v2.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #528]
+; CHECK-GI-NEXT: mov v5.b[7], w10
; CHECK-GI-NEXT: mov v3.b[8], w8
; CHECK-GI-NEXT: ldr w8, [sp, #152]
-; CHECK-GI-NEXT: mov v2.b[8], w11
-; CHECK-GI-NEXT: mov v0.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #792]
-; CHECK-GI-NEXT: ldr w11, [sp, #280]
-; CHECK-GI-NEXT: mov v4.b[8], w10
-; CHECK-GI-NEXT: mov v1.b[9], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #920]
-; CHECK-GI-NEXT: mov v5.b[8], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #32]
-; CHECK-GI-NEXT: ldr w12, [sp, #1048]
-; CHECK-GI-NEXT: mov v3.b[9], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #160]
-; CHECK-GI-NEXT: mov v2.b[9], w11
-; CHECK-GI-NEXT: mov v0.b[10], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #800]
-; CHECK-GI-NEXT: ldr w11, [sp, #288]
-; CHECK-GI-NEXT: mov v4.b[9], w10
-; CHECK-GI-NEXT: mov v1.b[10], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #928]
-; CHECK-GI-NEXT: mov v5.b[9], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #40]
-; CHECK-GI-NEXT: ldr w12, [sp, #1056]
+; CHECK-GI-NEXT: ldr w10, [sp, #656]
+; CHECK-GI-NEXT: mov v4.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #280]
+; CHECK-GI-NEXT: mov v0.b[9], w11
+; CHECK-GI-NEXT: mov v1.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #408]
+; CHECK-GI-NEXT: ldr w11, [sp, #32]
+; CHECK-GI-NEXT: mov v2.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #536]
+; CHECK-GI-NEXT: mov v5.b[8], w10
+; CHECK-GI-NEXT: mov v3.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #160]
+; CHECK-GI-NEXT: ldr w10, [sp, #664]
+; CHECK-GI-NEXT: mov v4.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #288]
+; CHECK-GI-NEXT: mov v0.b[10], w11
+; CHECK-GI-NEXT: mov v1.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #416]
+; CHECK-GI-NEXT: ldr w11, [sp, #40]
+; CHECK-GI-NEXT: mov v2.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #544]
+; CHECK-GI-NEXT: mov v5.b[9], w10
; CHECK-GI-NEXT: mov v3.b[10], w8
; CHECK-GI-NEXT: ldr w8, [sp, #168]
-; CHECK-GI-NEXT: mov v2.b[10], w11
-; CHECK-GI-NEXT: mov v0.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #808]
-; CHECK-GI-NEXT: ldr w11, [sp, #296]
-; CHECK-GI-NEXT: mov v4.b[10], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #672]
+; CHECK-GI-NEXT: mov v4.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #296]
+; CHECK-GI-NEXT: mov v0.b[11], w11
; CHECK-GI-NEXT: mov v1.b[11], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #936]
-; CHECK-GI-NEXT: mov v5.b[10], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #48]
-; CHECK-GI-NEXT: ldr w12, [sp, #1064]
-; CHECK-GI-NEXT: mov v3.b[11], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #176]
-; CHECK-GI-NEXT: mov v2.b[11], w11
-; CHECK-GI-NEXT: mov v0.b[12], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #816]
-; CHECK-GI-NEXT: ldr w11, [sp, #304]
-; CHECK-GI-NEXT: mov v4.b[11], w10
-; CHECK-GI-NEXT: mov v1.b[12], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #944]
-; CHECK-GI-NEXT: mov v5.b[11], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #56]
-; CHECK-GI-NEXT: ldr w12, [sp, #1072]
+; CHECK-GI-NEXT: ldr w8, [sp, #424]
+; CHECK-GI-NEXT: ldr w11, [sp, #48]
+; CHECK-GI-NEXT: mov v2.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #552]
+; CHECK-GI-NEXT: mov v5.b[10], w10
+; CHECK-GI-NEXT: mov v3.b[11], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #176]
+; CHECK-GI-NEXT: ldr w10, [sp, #680]
+; CHECK-GI-NEXT: mov v4.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #304]
+; CHECK-GI-NEXT: mov v0.b[12], w11
+; CHECK-GI-NEXT: mov v1.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #432]
+; CHECK-GI-NEXT: ldr w11, [sp, #56]
+; CHECK-GI-NEXT: mov v2.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #560]
+; CHECK-GI-NEXT: mov v5.b[11], w10
; CHECK-GI-NEXT: mov v3.b[12], w8
; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: mov v2.b[12], w11
-; CHECK-GI-NEXT: mov v0.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #824]
-; CHECK-GI-NEXT: ldr w11, [sp, #312]
-; CHECK-GI-NEXT: mov v4.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #688]
+; CHECK-GI-NEXT: mov v4.b[12], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #312]
+; CHECK-GI-NEXT: mov v0.b[13], w11
; CHECK-GI-NEXT: mov v1.b[13], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #952]
-; CHECK-GI-NEXT: mov v5.b[12], w12
-; CHECK-GI-NEXT: ldr w8, [sp, #64]
-; CHECK-GI-NEXT: ldr w12, [sp, #1080]
-; CHECK-GI-NEXT: mov v3.b[13], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #192]
-; CHECK-GI-NEXT: mov v2.b[13], w11
-; CHECK-GI-NEXT: mov v0.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #832]
-; CHECK-GI-NEXT: ldr w11, [sp, #320]
-; CHECK-GI-NEXT: mov v4.b[13], w10
-; CHECK-GI-NEXT: mov v1.b[14], w9
-; CHECK-GI-NEXT: ldr w10, [sp, #960]
-; CHECK-GI-NEXT: mov v5.b[13], w12
-; CHECK-GI-NEXT: ldr w9, [sp, #72]
-; CHECK-GI-NEXT: ldr w12, [sp, #1088]
+; CHECK-GI-NEXT: ldr w8, [sp, #440]
+; CHECK-GI-NEXT: ldr w11, [sp, #64]
+; CHECK-GI-NEXT: mov v2.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #568]
+; CHECK-GI-NEXT: mov v5.b[12], w10
+; CHECK-GI-NEXT: mov v3.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #192]
+; CHECK-GI-NEXT: ldr w10, [sp, #696]
+; CHECK-GI-NEXT: mov v4.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #320]
+; CHECK-GI-NEXT: mov v0.b[14], w11
+; CHECK-GI-NEXT: mov v1.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #448]
+; CHECK-GI-NEXT: ldr w11, [sp, #72]
+; CHECK-GI-NEXT: mov v2.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #576]
+; CHECK-GI-NEXT: mov v5.b[13], w10
; CHECK-GI-NEXT: mov v3.b[14], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #200]
-; CHECK-GI-NEXT: mov v2.b[14], w11
-; CHECK-GI-NEXT: mov v0.b[15], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #840]
-; CHECK-GI-NEXT: ldr w11, [sp, #328]
-; CHECK-GI-NEXT: mov v4.b[14], w10
-; CHECK-GI-NEXT: mov v1.b[15], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #968]
-; CHECK-GI-NEXT: mov v5.b[14], w12
-; CHECK-GI-NEXT: ldr w10, [sp, #1096]
-; CHECK-GI-NEXT: mov v3.b[15], w9
-; CHECK-GI-NEXT: mov v2.b[15], w11
-; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b
-; CHECK-GI-NEXT: mov v4.b[15], w8
-; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b
+; CHECK-GI-NEXT: ldr w8, [sp, #720]
+; CHECK-GI-NEXT: ldr w10, [sp, #704]
+; CHECK-GI-NEXT: mov v4.b[14], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #728]
+; CHECK-GI-NEXT: mov v0.b[15], w11
+; CHECK-GI-NEXT: fmov s6, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #328]
+; CHECK-GI-NEXT: ldr w11, [sp, #456]
+; CHECK-GI-NEXT: mov v5.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #200]
+; CHECK-GI-NEXT: movi v25.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v2.b[15], w8
+; CHECK-GI-NEXT: mov v3.b[15], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #736]
+; CHECK-GI-NEXT: mov v6.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #584]
+; CHECK-GI-NEXT: ldr w8, [sp, #856]
+; CHECK-GI-NEXT: mov v1.b[15], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #712]
+; CHECK-GI-NEXT: mov v4.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #976]
+; CHECK-GI-NEXT: mov v7.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1232]
; CHECK-GI-NEXT: mov v5.b[15], w10
-; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b
-; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b
-; CHECK-GI-NEXT: addv s0, v7.4s
-; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b
-; CHECK-GI-NEXT: addv s1, v16.4s
-; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b
-; CHECK-GI-NEXT: addv s2, v17.4s
-; CHECK-GI-NEXT: addv s4, v20.4s
+; CHECK-GI-NEXT: ldr w10, [sp, #984]
+; CHECK-GI-NEXT: mov v6.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1104]
+; CHECK-GI-NEXT: fmov s16, w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1360]
+; CHECK-GI-NEXT: fmov s18, w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1368]
+; CHECK-GI-NEXT: fmov s17, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1240]
+; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b
+; CHECK-GI-NEXT: mov v16.b[1], w10
+; CHECK-GI-NEXT: fmov s19, w9
+; CHECK-GI-NEXT: ldr w10, [sp, #864]
+; CHECK-GI-NEXT: mov v18.b[1], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: ldr w9, [sp, #1120]
+; CHECK-GI-NEXT: mov v17.b[1], w12
+; CHECK-GI-NEXT: mov v7.b[2], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1248]
+; CHECK-GI-NEXT: mov v19.b[1], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b
+; CHECK-GI-NEXT: mov v16.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #872]
+; CHECK-GI-NEXT: addv s0, v20.4s
+; CHECK-GI-NEXT: mov v6.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1000]
+; CHECK-GI-NEXT: mov v18.b[2], w10
+; CHECK-GI-NEXT: mov v17.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1376]
+; CHECK-GI-NEXT: ldr w10, [sp, #1128]
+; CHECK-GI-NEXT: mov v7.b[3], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #880]
+; CHECK-GI-NEXT: addv s1, v21.4s
+; CHECK-GI-NEXT: mov v19.b[2], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #752]
+; CHECK-GI-NEXT: mov v16.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1256]
+; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b
+; CHECK-GI-NEXT: mov v17.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1384]
+; CHECK-GI-NEXT: mov v6.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1008]
+; CHECK-GI-NEXT: mov v18.b[3], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1136]
+; CHECK-GI-NEXT: mov v19.b[3], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #760]
+; CHECK-GI-NEXT: mov v7.b[4], w11
+; CHECK-GI-NEXT: mov v16.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1264]
+; CHECK-GI-NEXT: ldr w11, [sp, #888]
+; CHECK-GI-NEXT: mov v17.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1392]
+; CHECK-GI-NEXT: mov v6.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1016]
+; CHECK-GI-NEXT: mov v18.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1144]
+; CHECK-GI-NEXT: mov v19.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #768]
+; CHECK-GI-NEXT: mov v7.b[5], w11
+; CHECK-GI-NEXT: mov v16.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1272]
+; CHECK-GI-NEXT: ldr w11, [sp, #896]
+; CHECK-GI-NEXT: mov v17.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1400]
+; CHECK-GI-NEXT: mov v6.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1024]
+; CHECK-GI-NEXT: mov v18.b[5], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1152]
+; CHECK-GI-NEXT: mov v19.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #776]
+; CHECK-GI-NEXT: mov v7.b[6], w11
+; CHECK-GI-NEXT: mov v16.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1280]
+; CHECK-GI-NEXT: ldr w11, [sp, #904]
+; CHECK-GI-NEXT: mov v17.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1408]
+; CHECK-GI-NEXT: mov v6.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1032]
+; CHECK-GI-NEXT: mov v18.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1160]
+; CHECK-GI-NEXT: mov v19.b[6], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #784]
+; CHECK-GI-NEXT: mov v7.b[7], w11
+; CHECK-GI-NEXT: mov v16.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1288]
+; CHECK-GI-NEXT: ldr w11, [sp, #912]
+; CHECK-GI-NEXT: mov v17.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1416]
+; CHECK-GI-NEXT: mov v6.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1040]
+; CHECK-GI-NEXT: mov v18.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1168]
+; CHECK-GI-NEXT: mov v19.b[7], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #792]
+; CHECK-GI-NEXT: mov v7.b[8], w11
+; CHECK-GI-NEXT: mov v16.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1296]
+; CHECK-GI-NEXT: ldr w11, [sp, #920]
+; CHECK-GI-NEXT: mov v17.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1424]
+; CHECK-GI-NEXT: mov v6.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1048]
+; CHECK-GI-NEXT: mov v18.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1176]
+; CHECK-GI-NEXT: mov v19.b[8], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #800]
+; CHECK-GI-NEXT: mov v7.b[9], w11
+; CHECK-GI-NEXT: mov v16.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1304]
+; CHECK-GI-NEXT: ldr w11, [sp, #928]
+; CHECK-GI-NEXT: mov v17.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1432]
+; CHECK-GI-NEXT: mov v6.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1056]
+; CHECK-GI-NEXT: mov v18.b[9], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1184]
+; CHECK-GI-NEXT: mov v19.b[9], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #808]
+; CHECK-GI-NEXT: mov v7.b[10], w11
+; CHECK-GI-NEXT: mov v16.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1312]
+; CHECK-GI-NEXT: ldr w11, [sp, #936]
+; CHECK-GI-NEXT: mov v17.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1440]
+; CHECK-GI-NEXT: mov v6.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1064]
+; CHECK-GI-NEXT: mov v18.b[10], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1192]
+; CHECK-GI-NEXT: mov v19.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #816]
+; CHECK-GI-NEXT: mov v7.b[11], w11
+; CHECK-GI-NEXT: mov v16.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1320]
+; CHECK-GI-NEXT: ldr w11, [sp, #944]
+; CHECK-GI-NEXT: mov v17.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1448]
+; CHECK-GI-NEXT: mov v6.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1072]
+; CHECK-GI-NEXT: mov v18.b[11], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1200]
+; CHECK-GI-NEXT: mov v19.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #824]
+; CHECK-GI-NEXT: mov v7.b[12], w11
+; CHECK-GI-NEXT: mov v16.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1328]
+; CHECK-GI-NEXT: ldr w11, [sp, #952]
+; CHECK-GI-NEXT: mov v17.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1456]
+; CHECK-GI-NEXT: mov v6.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1080]
+; CHECK-GI-NEXT: mov v18.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1208]
+; CHECK-GI-NEXT: mov v19.b[12], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #832]
+; CHECK-GI-NEXT: mov v7.b[13], w11
+; CHECK-GI-NEXT: mov v16.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1336]
+; CHECK-GI-NEXT: ldr w11, [sp, #960]
+; CHECK-GI-NEXT: mov v17.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1464]
+; CHECK-GI-NEXT: mov v6.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1088]
+; CHECK-GI-NEXT: mov v18.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #1216]
+; CHECK-GI-NEXT: mov v19.b[13], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #840]
+; CHECK-GI-NEXT: mov v7.b[14], w11
+; CHECK-GI-NEXT: mov v16.b[14], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #1344]
+; CHECK-GI-NEXT: ldr w11, [sp, #968]
+; CHECK-GI-NEXT: mov v17.b[14], w9
+; CHECK-GI-NEXT: mov v6.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1096]
+; CHECK-GI-NEXT: mov v18.b[14], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #1472]
+; CHECK-GI-NEXT: ldr w10, [sp, #1224]
+; CHECK-GI-NEXT: mov v7.b[15], w11
+; CHECK-GI-NEXT: addv s4, v25.4s
+; CHECK-GI-NEXT: mov v16.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #1352]
+; CHECK-GI-NEXT: mov v19.b[14], w9
+; CHECK-GI-NEXT: mov v17.b[15], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #1480]
+; CHECK-GI-NEXT: mov v18.b[15], w8
; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: fmov w11, s4
+; CHECK-GI-NEXT: mov v19.b[15], w9
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: addv s3, v18.4s
-; CHECK-GI-NEXT: addv s5, v19.4s
-; CHECK-GI-NEXT: fmov w10, s2
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w9, s4
-; CHECK-GI-NEXT: fmov w11, s3
+; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b
+; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: fmov w11, s5
-; CHECK-GI-NEXT: add w9, w10, w11
+; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b
+; CHECK-GI-NEXT: add w8, w8, w11
+; CHECK-GI-NEXT: addv s2, v22.4s
+; CHECK-GI-NEXT: addv s3, v23.4s
+; CHECK-GI-NEXT: addv s5, v24.4s
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: fmov w10, s3
+; CHECK-GI-NEXT: add w9, w9, w10
+; CHECK-GI-NEXT: fmov w10, s5
+; CHECK-GI-NEXT: add w9, w9, w10
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%az = sext <48 x i8> %a to <48 x i32>
- %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
+ %bz = sext <48 x i8> %b to <48 x i32>
+ %m1 = mul nuw nsw <48 x i32> %az, %bz
+ %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
%cz = sext <48 x i8> %c to <48 x i32>
- %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
+ %dz = sext <48 x i8> %d to <48 x i32>
+ %m2 = mul nuw nsw <48 x i32> %cz, %dz
+ %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
%x = add i32 %r1, %r2
ret i32 %x
}
-define i32 @test_usdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-SD-LABEL: test_usdot_v48i8:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
-; CHECK-SD-NEXT: ldr q3, [x0, #32]
-; CHECK-SD-NEXT: ldp q7, q2, [x0]
-; CHECK-SD-NEXT: ldr q4, [x1]
-; CHECK-SD-NEXT: ldr q5, [x1, #32]
-; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
-; CHECK-SD-NEXT: usdot v0.4s, v3.16b, v5.16b
-; CHECK-SD-NEXT: usdot v1.4s, v7.16b, v4.16b
-; CHECK-SD-NEXT: ldr q3, [x1, #16]
-; CHECK-SD-NEXT: usdot v6.4s, v2.16b, v3.16b
-; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v6.4s
-; CHECK-SD-NEXT: addv s0, v0.4s
-; CHECK-SD-NEXT: fmov w8, s0
-; CHECK-SD-NEXT: add w0, w8, w2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: test_usdot_v48i8:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldp q0, q2, [x1]
-; CHECK-GI-NEXT: ldr q7, [x1, #32]
-; CHECK-GI-NEXT: ldp q1, q4, [x0]
-; CHECK-GI-NEXT: ldr q6, [x0, #32]
-; CHECK-GI-NEXT: sshll v20.8h, v7.8b, #0
-; CHECK-GI-NEXT: sshll v3.8h, v0.8b, #0
-; CHECK-GI-NEXT: sshll2 v18.8h, v0.16b, #0
-; CHECK-GI-NEXT: sshll v19.8h, v2.8b, #0
-; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0
-; CHECK-GI-NEXT: sshll2 v0.8h, v7.16b, #0
-; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT: ushll v7.8h, v4.8b, #0
-; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0
-; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0
-; CHECK-GI-NEXT: sshll v16.4s, v3.4h, #0
-; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT: sshll v21.4s, v19.4h, #0
-; CHECK-GI-NEXT: ushll v17.4s, v5.4h, #0
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v25.4s, v1.4h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0
-; CHECK-GI-NEXT: sshll v22.4s, v2.4h, #0
-; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT: ushll v27.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v5.4s
-; CHECK-GI-NEXT: mul v16.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT: sshll v17.4s, v18.4h, #0
-; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
-; CHECK-GI-NEXT: ushll v5.8h, v6.8b, #0
-; CHECK-GI-NEXT: ushll2 v6.8h, v6.16b, #0
-; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0
-; CHECK-GI-NEXT: sshll v23.4s, v20.4h, #0
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT: sshll v24.4s, v0.4h, #0
-; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0
-; CHECK-GI-NEXT: addv s3, v3.4s
-; CHECK-GI-NEXT: addv s16, v16.4s
-; CHECK-GI-NEXT: mul v1.4s, v18.4s, v1.4s
-; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0
-; CHECK-GI-NEXT: ushll v18.4s, v6.4h, #0
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: ushll2 v3.4s, v7.8h, #0
-; CHECK-GI-NEXT: mul v7.4s, v17.4s, v25.4s
-; CHECK-GI-NEXT: mul v17.4s, v21.4s, v26.4s
-; CHECK-GI-NEXT: fmov w8, s16
-; CHECK-GI-NEXT: ushll v16.4s, v5.4h, #0
-; CHECK-GI-NEXT: mul v21.4s, v22.4s, v27.4s
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: mul v3.4s, v19.4s, v3.4s
-; CHECK-GI-NEXT: mul v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT: fmov w12, s2
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: mul v6.4s, v23.4s, v16.4s
-; CHECK-GI-NEXT: mul v16.4s, v24.4s, v18.4s
-; CHECK-GI-NEXT: addv s17, v17.4s
-; CHECK-GI-NEXT: fmov w10, s1
-; CHECK-GI-NEXT: mul v5.4s, v20.4s, v5.4s
-; CHECK-GI-NEXT: addv s4, v21.4s
-; CHECK-GI-NEXT: addv s7, v7.4s
-; CHECK-GI-NEXT: addv s1, v3.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: fmov w11, s17
-; CHECK-GI-NEXT: addv s3, v6.4s
-; CHECK-GI-NEXT: fmov w9, s7
-; CHECK-GI-NEXT: addv s2, v5.4s
-; CHECK-GI-NEXT: fmov w13, s1
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: fmov w11, s4
-; CHECK-GI-NEXT: addv s4, v16.4s
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w9, w10, w13
-; CHECK-GI-NEXT: fmov w13, s0
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: fmov w12, s3
-; CHECK-GI-NEXT: add w10, w11, w12
-; CHECK-GI-NEXT: fmov w11, s2
-; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: add w9, w10, w11
-; CHECK-GI-NEXT: add w10, w12, w13
-; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w9, w10, w2
-; CHECK-GI-NEXT: add w0, w8, w9
-; CHECK-GI-NEXT: ret
-entry:
- %0 = load <48 x i8>, ptr %a
- %1 = zext <48 x i8> %0 to <48 x i32>
- %2 = load <48 x i8>, ptr %b
- %3 = sext <48 x i8> %2 to <48 x i32>
- %4 = mul nsw <48 x i32> %3, %1
- %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)
- %op.extra = add nsw i32 %5, %sum
- ret i32 %op.extra
-}
-
-define i32 @test_usdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
-; CHECK-SD-LABEL: test_usdot_v48i8_double:
+define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
+; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: .cfi_offset w29, -16
-; CHECK-SD-NEXT: ldr b2, [sp, #592]
-; CHECK-SD-NEXT: add x8, sp, #600
; CHECK-SD-NEXT: ldr b5, [sp, #208]
-; CHECK-SD-NEXT: add x9, sp, #616
-; CHECK-SD-NEXT: ldr b0, [sp, #336]
-; CHECK-SD-NEXT: add x10, sp, #624
-; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]
; CHECK-SD-NEXT: add x8, sp, #216
-; CHECK-SD-NEXT: add x11, sp, #240
+; CHECK-SD-NEXT: fmov s0, w0
+; CHECK-SD-NEXT: ldr b4, [sp, #976]
+; CHECK-SD-NEXT: add x9, sp, #984
+; CHECK-SD-NEXT: add x12, sp, #328
; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #608
-; CHECK-SD-NEXT: ldr b4, [sp, #464]
-; CHECK-SD-NEXT: ldr b6, [sp, #80]
-; CHECK-SD-NEXT: ldr b7, [sp, #1360]
-; CHECK-SD-NEXT: ldr b16, [sp, #976]
-; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8]
; CHECK-SD-NEXT: add x8, sp, #224
-; CHECK-SD-NEXT: fmov s1, w0
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #344
+; CHECK-SD-NEXT: movi v1.16b, #1
+; CHECK-SD-NEXT: mov v0.b[1], w1
+; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9]
; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #232
-; CHECK-SD-NEXT: ldr b18, [sp, #720]
-; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #632
-; CHECK-SD-NEXT: mov v1.b[1], w1
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
-; CHECK-SD-NEXT: add x8, sp, #472
-; CHECK-SD-NEXT: add x12, sp, #856
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8]
-; CHECK-SD-NEXT: add x8, sp, #640
-; CHECK-SD-NEXT: add x13, sp, #864
-; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10]
-; CHECK-SD-NEXT: add x10, sp, #88
-; CHECK-SD-NEXT: movi v17.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[1], [x10]
-; CHECK-SD-NEXT: add x10, sp, #248
-; CHECK-SD-NEXT: add x11, sp, #272
-; CHECK-SD-NEXT: mov v1.b[2], w2
-; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1368
-; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #984
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[1], [x9]
-; CHECK-SD-NEXT: add x9, sp, #352
-; CHECK-SD-NEXT: add x10, sp, #648
-; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v0.b }[2], [x9]
-; CHECK-SD-NEXT: add x9, sp, #256
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT: add x8, sp, #480
-; CHECK-SD-NEXT: add x9, sp, #96
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x8]
-; CHECK-SD-NEXT: add x8, sp, #656
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9]
-; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10]
-; CHECK-SD-NEXT: add x9, sp, #264
-; CHECK-SD-NEXT: add x10, sp, #1376
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #664
-; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #992
-; CHECK-SD-NEXT: mov v1.b[3], w3
-; CHECK-SD-NEXT: ld1 { v2.b }[8], [x8]
-; CHECK-SD-NEXT: add x8, sp, #360
-; CHECK-SD-NEXT: ld1 { v16.b }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11]
-; CHECK-SD-NEXT: add x11, sp, #488
-; CHECK-SD-NEXT: add x10, sp, #672
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11]
-; CHECK-SD-NEXT: add x11, sp, #280
-; CHECK-SD-NEXT: ld1 { v2.b }[9], [x9]
-; CHECK-SD-NEXT: add x9, sp, #104
-; CHECK-SD-NEXT: add x8, sp, #680
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #368
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
-; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1384
-; CHECK-SD-NEXT: add x11, sp, #688
-; CHECK-SD-NEXT: ld1 { v2.b }[10], [x10]
-; CHECK-SD-NEXT: add x10, sp, #376
-; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9]
-; CHECK-SD-NEXT: add x9, sp, #288
-; CHECK-SD-NEXT: mov v1.b[4], w4
-; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10]
-; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
-; CHECK-SD-NEXT: add x10, sp, #296
-; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #496
-; CHECK-SD-NEXT: add x9, sp, #1000
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #384
-; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9]
-; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8]
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x10]
-; CHECK-SD-NEXT: add x8, sp, #112
-; CHECK-SD-NEXT: ld1 { v2.b }[12], [x11]
-; CHECK-SD-NEXT: add x11, sp, #392
-; CHECK-SD-NEXT: add x9, sp, #696
-; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8]
-; CHECK-SD-NEXT: add x8, sp, #304
-; CHECK-SD-NEXT: add x10, sp, #704
-; CHECK-SD-NEXT: ld1 { v0.b }[7], [x11]
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
-; CHECK-SD-NEXT: add x11, sp, #312
-; CHECK-SD-NEXT: ld1 { v2.b }[13], [x9]
-; CHECK-SD-NEXT: add x9, sp, #504
-; CHECK-SD-NEXT: add x8, sp, #1392
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x9]
-; CHECK-SD-NEXT: add x9, sp, #400
-; CHECK-SD-NEXT: ld1 { v7.b }[4], [x8]
-; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x11]
-; CHECK-SD-NEXT: add x11, sp, #320
-; CHECK-SD-NEXT: ld1 { v2.b }[14], [x10]
-; CHECK-SD-NEXT: add x10, sp, #120
-; CHECK-SD-NEXT: add x9, sp, #328
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10]
-; CHECK-SD-NEXT: add x10, sp, #408
-; CHECK-SD-NEXT: add x8, sp, #712
-; CHECK-SD-NEXT: ld1 { v0.b }[9], [x10]
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x11]
-; CHECK-SD-NEXT: add x11, sp, #416
-; CHECK-SD-NEXT: add x10, sp, #512
-; CHECK-SD-NEXT: ld1 { v2.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #128
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1016
-; CHECK-SD-NEXT: ld1 { v0.b }[10], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1400
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1008
-; CHECK-SD-NEXT: ld1 { v7.b }[5], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1408
-; CHECK-SD-NEXT: ld1 { v16.b }[4], [x9]
-; CHECK-SD-NEXT: add x9, sp, #520
-; CHECK-SD-NEXT: add x10, sp, #424
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9]
-; CHECK-SD-NEXT: add x9, sp, #136
-; CHECK-SD-NEXT: ld1 { v0.b }[11], [x10]
-; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9]
-; CHECK-SD-NEXT: add x10, sp, #1416
-; CHECK-SD-NEXT: ld1 { v16.b }[5], [x8]
-; CHECK-SD-NEXT: add x8, sp, #528
-; CHECK-SD-NEXT: add x9, sp, #144
-; CHECK-SD-NEXT: ld1 { v4.b }[8], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1024
-; CHECK-SD-NEXT: add x11, sp, #1432
-; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x9]
-; CHECK-SD-NEXT: add x10, sp, #1424
-; CHECK-SD-NEXT: ld1 { v16.b }[6], [x8]
-; CHECK-SD-NEXT: add x8, sp, #536
-; CHECK-SD-NEXT: add x9, sp, #152
-; CHECK-SD-NEXT: ld1 { v4.b }[9], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1032
-; CHECK-SD-NEXT: usdot v3.4s, v5.16b, v2.16b
-; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
-; CHECK-SD-NEXT: add x10, sp, #160
-; CHECK-SD-NEXT: ld1 { v16.b }[7], [x8]
-; CHECK-SD-NEXT: add x8, sp, #544
-; CHECK-SD-NEXT: add x9, sp, #432
-; CHECK-SD-NEXT: ld1 { v4.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1040
-; CHECK-SD-NEXT: ld1 { v0.b }[12], [x9]
-; CHECK-SD-NEXT: ld1 { v7.b }[9], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x10]
-; CHECK-SD-NEXT: add x11, sp, #1440
-; CHECK-SD-NEXT: ld1 { v16.b }[8], [x8]
-; CHECK-SD-NEXT: add x8, sp, #552
-; CHECK-SD-NEXT: add x10, sp, #168
-; CHECK-SD-NEXT: ld1 { v4.b }[11], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1048
-; CHECK-SD-NEXT: add x9, sp, #176
-; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1448
-; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8]
-; CHECK-SD-NEXT: add x8, sp, #560
-; CHECK-SD-NEXT: add x11, sp, #576
-; CHECK-SD-NEXT: ld1 { v4.b }[12], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1056
-; CHECK-SD-NEXT: ldr b5, [sp, #1104]
-; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9]
-; CHECK-SD-NEXT: add x10, sp, #1456
-; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8]
-; CHECK-SD-NEXT: add x8, sp, #568
-; CHECK-SD-NEXT: add x9, sp, #184
-; CHECK-SD-NEXT: ld1 { v4.b }[13], [x8]
-; CHECK-SD-NEXT: add x8, sp, #1064
-; CHECK-SD-NEXT: mov v1.b[5], w5
-; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1072
-; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8]
-; CHECK-SD-NEXT: add x10, sp, #192
+; CHECK-SD-NEXT: add x11, sp, #992
+; CHECK-SD-NEXT: ldr b6, [sp, #720]
+; CHECK-SD-NEXT: ldr b7, [sp, #80]
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT: add x8, sp, #232
+; CHECK-SD-NEXT: add x13, sp, #88
+; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11]
+; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13]
+; CHECK-SD-NEXT: add x13, sp, #856
+; CHECK-SD-NEXT: mov v0.b[2], w2
+; CHECK-SD-NEXT: add x14, sp, #1008
+; CHECK-SD-NEXT: add x15, sp, #872
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT: add x8, sp, #240
+; CHECK-SD-NEXT: add x16, sp, #888
+; CHECK-SD-NEXT: add x10, sp, #16
+; CHECK-SD-NEXT: add x9, sp, #24
+; CHECK-SD-NEXT: add x11, sp, #40
; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v4.b }[14], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1464
-; CHECK-SD-NEXT: add x8, sp, #440
-; CHECK-SD-NEXT: ld1 { v7.b }[13], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10]
-; CHECK-SD-NEXT: add x11, sp, #1472
-; CHECK-SD-NEXT: ld1 { v16.b }[12], [x9]
-; CHECK-SD-NEXT: add x9, sp, #584
-; CHECK-SD-NEXT: add x10, sp, #200
-; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1080
-; CHECK-SD-NEXT: mov v1.b[6], w6
-; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11]
-; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1112
-; CHECK-SD-NEXT: ld1 { v16.b }[13], [x9]
-; CHECK-SD-NEXT: add x11, sp, #1480
-; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10]
-; CHECK-SD-NEXT: add x9, sp, #1088
-; CHECK-SD-NEXT: add x10, sp, #1120
-; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
-; CHECK-SD-NEXT: ld1 { v7.b }[15], [x11]
-; CHECK-SD-NEXT: add x11, sp, #728
-; CHECK-SD-NEXT: usdot v2.4s, v6.16b, v4.16b
-; CHECK-SD-NEXT: ld1 { v18.b }[1], [x11]
-; CHECK-SD-NEXT: ld1 { v16.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v5.b }[2], [x10]
-; CHECK-SD-NEXT: add x10, sp, #736
-; CHECK-SD-NEXT: add x9, sp, #1096
-; CHECK-SD-NEXT: ldr b4, [sp, #1232]
-; CHECK-SD-NEXT: ldr b6, [sp, #848]
-; CHECK-SD-NEXT: add x11, sp, #1240
-; CHECK-SD-NEXT: mov v1.b[7], w7
-; CHECK-SD-NEXT: ld1 { v18.b }[2], [x10]
-; CHECK-SD-NEXT: ld1 { v16.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1128
-; CHECK-SD-NEXT: ld1 { v5.b }[3], [x9]
-; CHECK-SD-NEXT: add x10, sp, #744
-; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11]
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT: add x8, sp, #248
+; CHECK-SD-NEXT: mov v0.b[3], w3
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT: add x8, sp, #256
+; CHECK-SD-NEXT: mov v0.b[4], w4
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT: add x8, sp, #264
+; CHECK-SD-NEXT: mov v0.b[5], w5
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8]
+; CHECK-SD-NEXT: add x8, sp, #272
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8]
+; CHECK-SD-NEXT: add x8, sp, #280
+; CHECK-SD-NEXT: mov v0.b[6], w6
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8]
+; CHECK-SD-NEXT: add x8, sp, #288
+; CHECK-SD-NEXT: mov v0.b[7], w7
+; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #296
+; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10]
+; CHECK-SD-NEXT: add x10, sp, #128
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #304
+; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9]
+; CHECK-SD-NEXT: add x9, sp, #136
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #312
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #320
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT: add x8, sp, #32
+; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8]
+; CHECK-SD-NEXT: add x8, sp, #144
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12]
+; CHECK-SD-NEXT: add x12, sp, #728
; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1136
-; CHECK-SD-NEXT: add x12, sp, #1248
-; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10]
-; CHECK-SD-NEXT: add x11, sp, #752
-; CHECK-SD-NEXT: add x10, sp, #16
-; CHECK-SD-NEXT: ld1 { v5.b }[4], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1144
-; CHECK-SD-NEXT: ld1 { v6.b }[2], [x13]
-; CHECK-SD-NEXT: add x12, sp, #872
-; CHECK-SD-NEXT: ld1 { v1.b }[8], [x10]
-; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1256
-; CHECK-SD-NEXT: add x10, sp, #760
-; CHECK-SD-NEXT: ld1 { v5.b }[5], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11]
-; CHECK-SD-NEXT: add x9, sp, #1152
-; CHECK-SD-NEXT: ld1 { v6.b }[3], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1264
-; CHECK-SD-NEXT: add x13, sp, #880
-; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10]
-; CHECK-SD-NEXT: add x11, sp, #768
-; CHECK-SD-NEXT: add x10, sp, #24
-; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[4], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1160
+; CHECK-SD-NEXT: add x12, sp, #1000
+; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11]
+; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12]
+; CHECK-SD-NEXT: add x12, sp, #736
+; CHECK-SD-NEXT: add x11, sp, #920
+; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b
+; CHECK-SD-NEXT: ldr b5, [sp, #848]
+; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12]
+; CHECK-SD-NEXT: add x12, sp, #48
+; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13]
+; CHECK-SD-NEXT: add x13, sp, #744
+; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14]
+; CHECK-SD-NEXT: add x14, sp, #96
+; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12]
+; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13]
+; CHECK-SD-NEXT: add x13, sp, #864
+; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14]
+; CHECK-SD-NEXT: add x14, sp, #1016
+; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13]
+; CHECK-SD-NEXT: add x13, sp, #752
+; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14]
+; CHECK-SD-NEXT: add x14, sp, #104
; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13]
-; CHECK-SD-NEXT: add x12, sp, #888
-; CHECK-SD-NEXT: ld1 { v1.b }[9], [x10]
-; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1272
-; CHECK-SD-NEXT: add x10, sp, #776
-; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[5], [x11]
-; CHECK-SD-NEXT: add x9, sp, #1168
-; CHECK-SD-NEXT: ld1 { v6.b }[5], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1280
-; CHECK-SD-NEXT: add x13, sp, #896
-; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10]
-; CHECK-SD-NEXT: add x11, sp, #784
-; CHECK-SD-NEXT: add x10, sp, #32
-; CHECK-SD-NEXT: ld1 { v5.b }[8], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[6], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1176
-; CHECK-SD-NEXT: ld1 { v6.b }[6], [x13]
-; CHECK-SD-NEXT: add x12, sp, #904
-; CHECK-SD-NEXT: ld1 { v1.b }[10], [x10]
-; CHECK-SD-NEXT: ld1 { v18.b }[8], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1288
-; CHECK-SD-NEXT: add x10, sp, #792
-; CHECK-SD-NEXT: ld1 { v5.b }[9], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[7], [x11]
-; CHECK-SD-NEXT: add x9, sp, #1184
-; CHECK-SD-NEXT: ld1 { v6.b }[7], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1296
+; CHECK-SD-NEXT: add x13, sp, #1024
+; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14]
+; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15]
+; CHECK-SD-NEXT: add x15, sp, #760
+; CHECK-SD-NEXT: add x14, sp, #112
+; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13]
+; CHECK-SD-NEXT: add x13, sp, #880
+; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15]
+; CHECK-SD-NEXT: add x15, sp, #1032
+; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14]
+; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13]
+; CHECK-SD-NEXT: add x14, sp, #768
+; CHECK-SD-NEXT: add x13, sp, #120
+; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15]
+; CHECK-SD-NEXT: add x15, sp, #1040
+; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14]
+; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13]
+; CHECK-SD-NEXT: add x13, sp, #776
+; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16]
+; CHECK-SD-NEXT: add x14, sp, #1048
+; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15]
+; CHECK-SD-NEXT: add x15, sp, #896
+; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13]
+; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10]
+; CHECK-SD-NEXT: add x10, sp, #784
+; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15]
+; CHECK-SD-NEXT: add x13, sp, #1056
+; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14]
+; CHECK-SD-NEXT: add x14, sp, #904
+; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10]
+; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9]
+; CHECK-SD-NEXT: add x9, sp, #792
+; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14]
+; CHECK-SD-NEXT: add x10, sp, #1064
+; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13]
; CHECK-SD-NEXT: add x13, sp, #912
-; CHECK-SD-NEXT: ld1 { v18.b }[9], [x10]
-; CHECK-SD-NEXT: add x11, sp, #800
-; CHECK-SD-NEXT: add x10, sp, #40
+; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8]
+; CHECK-SD-NEXT: add x9, sp, #800
+; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13]
+; CHECK-SD-NEXT: add x8, sp, #152
+; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1072
+; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9]
+; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8]
+; CHECK-SD-NEXT: add x9, sp, #808
+; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11]
+; CHECK-SD-NEXT: add x8, sp, #56
+; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10]
+; CHECK-SD-NEXT: add x10, sp, #160
+; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9]
+; CHECK-SD-NEXT: add x9, sp, #928
+; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1080
; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[8], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1192
-; CHECK-SD-NEXT: ld1 { v6.b }[8], [x13]
-; CHECK-SD-NEXT: add x12, sp, #920
-; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10]
-; CHECK-SD-NEXT: ld1 { v18.b }[10], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1304
-; CHECK-SD-NEXT: add x10, sp, #808
-; CHECK-SD-NEXT: ld1 { v5.b }[11], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[9], [x11]
-; CHECK-SD-NEXT: add x9, sp, #1200
-; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1312
-; CHECK-SD-NEXT: add x13, sp, #928
-; CHECK-SD-NEXT: ld1 { v18.b }[11], [x10]
-; CHECK-SD-NEXT: add x11, sp, #816
-; CHECK-SD-NEXT: add x10, sp, #48
-; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[10], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1208
-; CHECK-SD-NEXT: ld1 { v6.b }[10], [x13]
-; CHECK-SD-NEXT: add x12, sp, #936
-; CHECK-SD-NEXT: ld1 { v1.b }[12], [x10]
-; CHECK-SD-NEXT: ld1 { v18.b }[12], [x11]
-; CHECK-SD-NEXT: add x11, sp, #1320
-; CHECK-SD-NEXT: add x10, sp, #824
-; CHECK-SD-NEXT: ld1 { v5.b }[13], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[11], [x11]
-; CHECK-SD-NEXT: add x9, sp, #1216
-; CHECK-SD-NEXT: ld1 { v6.b }[11], [x12]
-; CHECK-SD-NEXT: add x12, sp, #1328
-; CHECK-SD-NEXT: add x13, sp, #944
-; CHECK-SD-NEXT: ld1 { v18.b }[13], [x10]
-; CHECK-SD-NEXT: add x11, sp, #832
-; CHECK-SD-NEXT: add x10, sp, #56
-; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v4.b }[12], [x12]
-; CHECK-SD-NEXT: add x9, sp, #1224
-; CHECK-SD-NEXT: ld1 { v6.b }[12], [x13]
-; CHECK-SD-NEXT: ld1 { v1.b }[13], [x10]
-; CHECK-SD-NEXT: add x10, sp, #1336
-; CHECK-SD-NEXT: ld1 { v18.b }[14], [x11]
-; CHECK-SD-NEXT: add x11, sp, #952
-; CHECK-SD-NEXT: add x8, sp, #448
-; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #840
+; CHECK-SD-NEXT: add x8, sp, #816
; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10]
-; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11]
-; CHECK-SD-NEXT: usdot v17.4s, v16.16b, v7.16b
-; CHECK-SD-NEXT: movi v7.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v18.b }[15], [x9]
-; CHECK-SD-NEXT: add x9, sp, #64
-; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
-; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9]
-; CHECK-SD-NEXT: add x9, sp, #1344
-; CHECK-SD-NEXT: add x10, sp, #960
+; CHECK-SD-NEXT: add x9, sp, #168
+; CHECK-SD-NEXT: add x10, sp, #176
+; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8]
+; CHECK-SD-NEXT: add x8, sp, #936
+; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9]
+; CHECK-SD-NEXT: add x9, sp, #1088
+; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT: add x8, sp, #64
; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10]
-; CHECK-SD-NEXT: add x8, sp, #456
-; CHECK-SD-NEXT: usdot v7.4s, v18.16b, v5.16b
-; CHECK-SD-NEXT: movi v5.2d, #0000000000000000
-; CHECK-SD-NEXT: ld1 { v0.b }[15], [x8]
-; CHECK-SD-NEXT: add x8, sp, #72
-; CHECK-SD-NEXT: movi v16.2d, #0000000000000000
-; CHECK-SD-NEXT: add x9, sp, #1352
-; CHECK-SD-NEXT: add x10, sp, #968
-; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8]
-; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9]
-; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10]
-; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v0.16b
+; CHECK-SD-NEXT: add x9, sp, #824
+; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8]
+; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9]
+; CHECK-SD-NEXT: add x9, sp, #944
+; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10]
+; CHECK-SD-NEXT: add x10, sp, #1096
+; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9]
+; CHECK-SD-NEXT: add x8, sp, #832
+; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10]
+; CHECK-SD-NEXT: add x9, sp, #184
+; CHECK-SD-NEXT: add x10, sp, #72
+; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8]
+; CHECK-SD-NEXT: add x8, sp, #952
+; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT: add x8, sp, #840
+; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10]
+; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b
+; CHECK-SD-NEXT: add x9, sp, #192
+; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8]
+; CHECK-SD-NEXT: add x8, sp, #960
+; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9]
+; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8]
+; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT: add x8, sp, #200
+; CHECK-SD-NEXT: add x9, sp, #968
+; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b
+; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8]
+; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9]
+; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b
+; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b
; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s
-; CHECK-SD-NEXT: add v1.4s, v7.4s, v17.4s
-; CHECK-SD-NEXT: usdot v5.4s, v6.16b, v4.16b
-; CHECK-SD-NEXT: add v0.4s, v16.4s, v0.4s
-; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s
-; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SD-NEXT: ret
;
-; CHECK-GI-LABEL: test_usdot_v48i8_double:
+; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: .cfi_offset w29, -16
-; CHECK-GI-NEXT: ldr w8, [sp, #16]
-; CHECK-GI-NEXT: ldr w9, [sp, #80]
-; CHECK-GI-NEXT: fmov s2, w0
-; CHECK-GI-NEXT: ldr w10, [sp, #24]
-; CHECK-GI-NEXT: ldr w11, [sp, #144]
-; CHECK-GI-NEXT: ldr w12, [sp, #296]
-; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: ldr w10, [sp, #80]
+; CHECK-GI-NEXT: ldr w11, [sp, #208]
+; CHECK-GI-NEXT: fmov s0, w0
+; CHECK-GI-NEXT: ldr w9, [sp, #88]
+; CHECK-GI-NEXT: ldr w12, [sp, #728]
+; CHECK-GI-NEXT: movi v6.16b, #1
+; CHECK-GI-NEXT: fmov s1, w10
+; CHECK-GI-NEXT: fmov s2, w11
+; CHECK-GI-NEXT: ldr w11, [sp, #720]
+; CHECK-GI-NEXT: ldr w10, [sp, #216]
+; CHECK-GI-NEXT: mov v0.b[1], w1
+; CHECK-GI-NEXT: ldr w13, [sp, #856]
+; CHECK-GI-NEXT: fmov s3, w11
+; CHECK-GI-NEXT: ldr w8, [sp, #96]
+; CHECK-GI-NEXT: ldr w11, [sp, #224]
+; CHECK-GI-NEXT: mov v1.b[1], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #848]
+; CHECK-GI-NEXT: mov v2.b[1], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #976]
+; CHECK-GI-NEXT: movi v7.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v16.2d, #0000000000000000
; CHECK-GI-NEXT: fmov s4, w9
-; CHECK-GI-NEXT: ldr w8, [sp, #88]
-; CHECK-GI-NEXT: fmov s5, w11
-; CHECK-GI-NEXT: ldr w9, [sp, #152]
-; CHECK-GI-NEXT: ldr w11, [sp, #216]
-; CHECK-GI-NEXT: ldr w13, [sp, #64]
-; CHECK-GI-NEXT: ldr w14, [sp, #128]
-; CHECK-GI-NEXT: mov v2.b[1], w1
-; CHECK-GI-NEXT: mov v3.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #208]
-; CHECK-GI-NEXT: mov v4.b[1], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #32]
+; CHECK-GI-NEXT: mov v3.b[1], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #984]
+; CHECK-GI-NEXT: fmov s5, w10
+; CHECK-GI-NEXT: mov v0.b[2], w2
+; CHECK-GI-NEXT: ldr w10, [sp, #736]
+; CHECK-GI-NEXT: mov v1.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #864]
+; CHECK-GI-NEXT: mov v2.b[2], w11
+; CHECK-GI-NEXT: mov v4.b[1], w13
+; CHECK-GI-NEXT: ldr w11, [sp, #992]
+; CHECK-GI-NEXT: ldr w12, [sp, #776]
; CHECK-GI-NEXT: mov v5.b[1], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #96]
-; CHECK-GI-NEXT: fmov s6, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #224]
-; CHECK-GI-NEXT: ldr w15, [sp, #264]
-; CHECK-GI-NEXT: ldr w16, [sp, #528]
-; CHECK-GI-NEXT: mov v2.b[2], w2
-; CHECK-GI-NEXT: ldr w17, [sp, #648]
-; CHECK-GI-NEXT: mov v3.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #160]
-; CHECK-GI-NEXT: mov v4.b[2], w9
-; CHECK-GI-NEXT: mov v6.b[1], w11
-; CHECK-GI-NEXT: ldr w9, [sp, #272]
-; CHECK-GI-NEXT: ldr w11, [sp, #48]
-; CHECK-GI-NEXT: mov v5.b[2], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #40]
-; CHECK-GI-NEXT: fmov s18, w16
-; CHECK-GI-NEXT: fmov s0, w9
-; CHECK-GI-NEXT: ldr w9, [sp, #168]
-; CHECK-GI-NEXT: mov v2.b[3], w3
+; CHECK-GI-NEXT: mov v3.b[2], w10
+; CHECK-GI-NEXT: ldr w9, [sp, #104]
+; CHECK-GI-NEXT: ldr w10, [sp, #232]
+; CHECK-GI-NEXT: mov v0.b[3], w3
+; CHECK-GI-NEXT: movi v17.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v1.b[3], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #872]
+; CHECK-GI-NEXT: movi v18.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v4.b[2], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #744]
+; CHECK-GI-NEXT: mov v2.b[3], w10
+; CHECK-GI-NEXT: mov v5.b[2], w11
+; CHECK-GI-NEXT: ldr w11, [sp, #1000]
+; CHECK-GI-NEXT: ldr w10, [sp, #240]
; CHECK-GI-NEXT: mov v3.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #232]
-; CHECK-GI-NEXT: ldr w16, [sp, #584]
-; CHECK-GI-NEXT: mov v6.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #104]
-; CHECK-GI-NEXT: mov v5.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #56]
-; CHECK-GI-NEXT: mov v4.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #280]
-; CHECK-GI-NEXT: mov v2.b[4], w4
-; CHECK-GI-NEXT: mov v3.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #240]
-; CHECK-GI-NEXT: mov v6.b[3], w8
; CHECK-GI-NEXT: ldr w8, [sp, #112]
-; CHECK-GI-NEXT: mov v0.b[1], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #176]
-; CHECK-GI-NEXT: mov v4.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #288]
-; CHECK-GI-NEXT: mov v2.b[5], w5
-; CHECK-GI-NEXT: mov v5.b[4], w10
-; CHECK-GI-NEXT: mov v3.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #248]
-; CHECK-GI-NEXT: mov v0.b[2], w8
-; CHECK-GI-NEXT: mov v6.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #120]
-; CHECK-GI-NEXT: ldr w8, [sp, #184]
-; CHECK-GI-NEXT: ldr w10, [sp, #72]
-; CHECK-GI-NEXT: mov v4.b[5], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #136]
-; CHECK-GI-NEXT: mov v2.b[6], w6
-; CHECK-GI-NEXT: mov v5.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #192]
-; CHECK-GI-NEXT: mov v3.b[6], w13
-; CHECK-GI-NEXT: mov v0.b[3], w12
-; CHECK-GI-NEXT: mov v6.b[5], w9
-; CHECK-GI-NEXT: ldr w13, [sp, #304]
-; CHECK-GI-NEXT: ldr w9, [sp, #256]
-; CHECK-GI-NEXT: ldr w12, [sp, #200]
-; CHECK-GI-NEXT: mov v4.b[6], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #336]
-; CHECK-GI-NEXT: mov v2.b[7], w7
-; CHECK-GI-NEXT: mov v5.b[6], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #344]
-; CHECK-GI-NEXT: mov v3.b[7], w10
-; CHECK-GI-NEXT: mov v0.b[4], w13
-; CHECK-GI-NEXT: fmov s7, w14
-; CHECK-GI-NEXT: mov v6.b[6], w9
-; CHECK-GI-NEXT: ldr w14, [sp, #312]
-; CHECK-GI-NEXT: ldr w13, [sp, #352]
-; CHECK-GI-NEXT: ldr w10, [sp, #360]
-; CHECK-GI-NEXT: mov v4.b[7], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #472]
-; CHECK-GI-NEXT: ldr w9, [sp, #320]
-; CHECK-GI-NEXT: mov v5.b[7], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #400]
-; CHECK-GI-NEXT: mov v7.b[1], w8
-; CHECK-GI-NEXT: mov v0.b[5], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #464]
-; CHECK-GI-NEXT: mov v6.b[7], w15
-; CHECK-GI-NEXT: fmov s16, w12
-; CHECK-GI-NEXT: ldr w15, [sp, #408]
-; CHECK-GI-NEXT: ldr w8, [sp, #328]
-; CHECK-GI-NEXT: fmov s17, w14
-; CHECK-GI-NEXT: ldr w14, [sp, #592]
-; CHECK-GI-NEXT: ldr w12, [sp, #368]
-; CHECK-GI-NEXT: mov v7.b[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #536]
-; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
-; CHECK-GI-NEXT: mov v16.b[1], w15
-; CHECK-GI-NEXT: fmov s19, w14
-; CHECK-GI-NEXT: ldr w15, [sp, #600]
-; CHECK-GI-NEXT: mov v18.b[1], w13
-; CHECK-GI-NEXT: ldr w14, [sp, #416]
-; CHECK-GI-NEXT: mov v17.b[1], w11
-; CHECK-GI-NEXT: mov v0.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #480]
-; CHECK-GI-NEXT: ldr w11, [sp, #376]
-; CHECK-GI-NEXT: mov v19.b[1], w15
-; CHECK-GI-NEXT: mov v7.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #544]
-; CHECK-GI-NEXT: mov v16.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #608]
-; CHECK-GI-NEXT: ldr w13, [sp, #384]
-; CHECK-GI-NEXT: mov v18.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #424]
-; CHECK-GI-NEXT: mov v17.b[2], w9
-; CHECK-GI-NEXT: mov v0.b[7], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #488]
-; CHECK-GI-NEXT: ldr w9, [sp, #392]
-; CHECK-GI-NEXT: mov v19.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #552]
-; CHECK-GI-NEXT: mov v7.b[4], w12
-; CHECK-GI-NEXT: mov v16.b[3], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #656]
-; CHECK-GI-NEXT: ldr w12, [sp, #616]
-; CHECK-GI-NEXT: mov v17.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #432]
-; CHECK-GI-NEXT: mov v18.b[3], w14
-; CHECK-GI-NEXT: fmov s1, w10
-; CHECK-GI-NEXT: ldr w14, [sp, #664]
-; CHECK-GI-NEXT: ldr w10, [sp, #440]
-; CHECK-GI-NEXT: mov v19.b[3], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #496]
-; CHECK-GI-NEXT: mov v7.b[5], w11
-; CHECK-GI-NEXT: mov v16.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #624]
-; CHECK-GI-NEXT: ldr w11, [sp, #504]
-; CHECK-GI-NEXT: mov v1.b[1], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #560]
-; CHECK-GI-NEXT: mov v17.b[4], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #448]
-; CHECK-GI-NEXT: ldr w15, [sp, #456]
-; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
-; CHECK-GI-NEXT: mov v18.b[4], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #672]
-; CHECK-GI-NEXT: mov v19.b[4], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #568]
-; CHECK-GI-NEXT: mov v16.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #632]
-; CHECK-GI-NEXT: mov v1.b[2], w14
-; CHECK-GI-NEXT: mov v17.b[5], w11
-; CHECK-GI-NEXT: mov v7.b[6], w13
-; CHECK-GI-NEXT: ldr w11, [sp, #512]
-; CHECK-GI-NEXT: ldr w13, [sp, #520]
-; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0
-; CHECK-GI-NEXT: mov v18.b[5], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #680]
-; CHECK-GI-NEXT: mov v19.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #576]
-; CHECK-GI-NEXT: mov v16.b[6], w12
-; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
-; CHECK-GI-NEXT: mov v1.b[3], w8
-; CHECK-GI-NEXT: ldr w8, [sp, #640]
-; CHECK-GI-NEXT: mov v17.b[6], w11
-; CHECK-GI-NEXT: mov v7.b[7], w9
-; CHECK-GI-NEXT: ushll v20.4s, v2.4h, #0
-; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT: mov v18.b[6], w10
-; CHECK-GI-NEXT: mov v19.b[6], w8
-; CHECK-GI-NEXT: ldr w10, [sp, #688]
-; CHECK-GI-NEXT: mov v16.b[7], w15
-; CHECK-GI-NEXT: ushll v21.4s, v3.4h, #0
-; CHECK-GI-NEXT: ushll2 v22.4s, v3.8h, #0
-; CHECK-GI-NEXT: mov v17.b[7], w13
-; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll2 v24.4s, v4.8h, #0
-; CHECK-GI-NEXT: sshll v7.8h, v7.8b, #0
-; CHECK-GI-NEXT: ushll v3.4s, v6.4h, #0
-; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0
-; CHECK-GI-NEXT: mov v18.b[7], w16
-; CHECK-GI-NEXT: mov v19.b[7], w17
-; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
-; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0
-; CHECK-GI-NEXT: mov v1.b[4], w10
-; CHECK-GI-NEXT: ldr w9, [sp, #696]
-; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
-; CHECK-GI-NEXT: ldr w14, [sp, #720]
-; CHECK-GI-NEXT: ldr w13, [sp, #728]
-; CHECK-GI-NEXT: ushll v25.4s, v5.4h, #0
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ldr w12, [sp, #704]
-; CHECK-GI-NEXT: sshll v6.8h, v18.8b, #0
-; CHECK-GI-NEXT: sshll v18.8h, v19.8b, #0
-; CHECK-GI-NEXT: sshll v19.4s, v7.4h, #0
-; CHECK-GI-NEXT: sshll2 v7.4s, v7.8h, #0
-; CHECK-GI-NEXT: sshll v26.4s, v16.4h, #0
-; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
-; CHECK-GI-NEXT: sshll v27.4s, v17.4h, #0
-; CHECK-GI-NEXT: mov v1.b[5], w9
-; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
-; CHECK-GI-NEXT: mul v19.4s, v20.4s, v19.4s
-; CHECK-GI-NEXT: sshll v28.4s, v6.4h, #0
-; CHECK-GI-NEXT: sshll2 v6.4s, v6.8h, #0
-; CHECK-GI-NEXT: mul v7.4s, v2.4s, v7.4s
-; CHECK-GI-NEXT: mul v20.4s, v21.4s, v26.4s
-; CHECK-GI-NEXT: mul v16.4s, v22.4s, v16.4s
-; CHECK-GI-NEXT: mul v21.4s, v23.4s, v27.4s
-; CHECK-GI-NEXT: fmov s2, w14
-; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0
-; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
-; CHECK-GI-NEXT: mul v5.4s, v5.4s, v6.4s
-; CHECK-GI-NEXT: mul v17.4s, v24.4s, v17.4s
-; CHECK-GI-NEXT: addv s6, v19.4s
-; CHECK-GI-NEXT: mov v1.b[6], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #736]
-; CHECK-GI-NEXT: addv s7, v7.4s
-; CHECK-GI-NEXT: mov v2.b[1], w13
-; CHECK-GI-NEXT: addv s16, v16.4s
-; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT: addv s18, v20.4s
-; CHECK-GI-NEXT: addv s19, v21.4s
-; CHECK-GI-NEXT: fmov w12, s6
-; CHECK-GI-NEXT: mul v22.4s, v25.4s, v28.4s
-; CHECK-GI-NEXT: addv s17, v17.4s
-; CHECK-GI-NEXT: fmov w13, s7
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v29.4s
-; CHECK-GI-NEXT: addv s5, v5.4s
-; CHECK-GI-NEXT: mov v2.b[2], w11
-; CHECK-GI-NEXT: fmov w14, s19
-; CHECK-GI-NEXT: ldr w10, [sp, #744]
+; CHECK-GI-NEXT: mov v0.b[4], w4
+; CHECK-GI-NEXT: movi v19.2d, #0000000000000000
+; CHECK-GI-NEXT: movi v20.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v4.b[3], w9
; CHECK-GI-NEXT: ldr w9, [sp, #752]
-; CHECK-GI-NEXT: ldr w8, [sp, #712]
-; CHECK-GI-NEXT: ldr w15, [sp, #864]
-; CHECK-GI-NEXT: add w11, w12, w13
-; CHECK-GI-NEXT: fmov w12, s18
-; CHECK-GI-NEXT: fmov w13, s16
-; CHECK-GI-NEXT: addv s20, v22.4s
-; CHECK-GI-NEXT: addv s3, v3.4s
-; CHECK-GI-NEXT: mov v1.b[7], w8
-; CHECK-GI-NEXT: mov v2.b[3], w10
-; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: add w12, w13, w14
-; CHECK-GI-NEXT: fmov w13, s17
-; CHECK-GI-NEXT: fmov w10, s20
-; CHECK-GI-NEXT: ldr w14, [sp, #784]
-; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll v22.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: add w12, w12, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #792]
-; CHECK-GI-NEXT: mov v2.b[4], w9
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s5
-; CHECK-GI-NEXT: addv s5, v4.4s
-; CHECK-GI-NEXT: ldr w9, [sp, #856]
-; CHECK-GI-NEXT: sshll v23.4s, v1.4h, #0
-; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: add w10, w10, w12
-; CHECK-GI-NEXT: fmov w12, s3
-; CHECK-GI-NEXT: fmov s3, w14
-; CHECK-GI-NEXT: ldr w14, [sp, #760]
-; CHECK-GI-NEXT: mul v22.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: add w12, w10, w12
-; CHECK-GI-NEXT: ldr w10, [sp, #848]
-; CHECK-GI-NEXT: mov v3.b[1], w13
-; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mov v2.b[5], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #920]
-; CHECK-GI-NEXT: fmov s4, w10
-; CHECK-GI-NEXT: ldr w10, [sp, #768]
-; CHECK-GI-NEXT: addv s1, v22.4s
-; CHECK-GI-NEXT: addv s0, v0.4s
-; CHECK-GI-NEXT: add w12, w12, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #800]
-; CHECK-GI-NEXT: mov v4.b[1], w9
-; CHECK-GI-NEXT: add w8, w11, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #912]
-; CHECK-GI-NEXT: mov v3.b[2], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #976]
-; CHECK-GI-NEXT: ldr w11, [sp, #984]
-; CHECK-GI-NEXT: fmov s5, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1048]
-; CHECK-GI-NEXT: mov v2.b[6], w10
-; CHECK-GI-NEXT: fmov s6, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #808]
-; CHECK-GI-NEXT: ldr w10, [sp, #832]
-; CHECK-GI-NEXT: mov v4.b[2], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #1040]
-; CHECK-GI-NEXT: ldr w9, [sp, #776]
-; CHECK-GI-NEXT: mov v5.b[1], w14
-; CHECK-GI-NEXT: mov v3.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1056]
-; CHECK-GI-NEXT: fmov s7, w15
-; CHECK-GI-NEXT: mov v6.b[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #872]
-; CHECK-GI-NEXT: ldr w14, [sp, #816]
-; CHECK-GI-NEXT: ldr w15, [sp, #992]
-; CHECK-GI-NEXT: mov v2.b[7], w9
-; CHECK-GI-NEXT: mov v4.b[3], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #824]
-; CHECK-GI-NEXT: ldr w9, [sp, #840]
-; CHECK-GI-NEXT: mov v7.b[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #928]
-; CHECK-GI-NEXT: mov v3.b[4], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1064]
-; CHECK-GI-NEXT: mov v6.b[2], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #1000]
-; CHECK-GI-NEXT: mov v5.b[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #880]
-; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
-; CHECK-GI-NEXT: mov v7.b[2], w13
-; CHECK-GI-NEXT: mov v4.b[4], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1104]
-; CHECK-GI-NEXT: ldr w13, [sp, #936]
-; CHECK-GI-NEXT: mov v3.b[5], w11
-; CHECK-GI-NEXT: mov v6.b[3], w15
-; CHECK-GI-NEXT: fmov s16, w12
-; CHECK-GI-NEXT: ldr w12, [sp, #944]
+; CHECK-GI-NEXT: mov v1.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #880]
+; CHECK-GI-NEXT: mov v5.b[3], w11
+; CHECK-GI-NEXT: mov v2.b[4], w10
+; CHECK-GI-NEXT: mov v3.b[4], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #120]
; CHECK-GI-NEXT: ldr w11, [sp, #1008]
-; CHECK-GI-NEXT: mov v5.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1112]
-; CHECK-GI-NEXT: ldr w15, [sp, #904]
-; CHECK-GI-NEXT: mov v7.b[3], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #888]
-; CHECK-GI-NEXT: mov v16.b[1], w13
-; CHECK-GI-NEXT: mov v3.b[6], w10
-; CHECK-GI-NEXT: ldr w13, [sp, #896]
-; CHECK-GI-NEXT: mov v4.b[5], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1072]
-; CHECK-GI-NEXT: mov v6.b[4], w11
-; CHECK-GI-NEXT: mov v5.b[4], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1120]
-; CHECK-GI-NEXT: ldr w11, [sp, #952]
-; CHECK-GI-NEXT: mov v7.b[4], w14
-; CHECK-GI-NEXT: ldr w10, [sp, #1016]
-; CHECK-GI-NEXT: ldr w14, [sp, #960]
-; CHECK-GI-NEXT: mov v16.b[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1080]
-; CHECK-GI-NEXT: mov v3.b[7], w9
-; CHECK-GI-NEXT: mov v4.b[6], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1128]
-; CHECK-GI-NEXT: ldr w9, [sp, #1088]
+; CHECK-GI-NEXT: ldr w10, [sp, #248]
+; CHECK-GI-NEXT: mov v0.b[5], w5
+; CHECK-GI-NEXT: mov v4.b[4], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #760]
+; CHECK-GI-NEXT: mov v1.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #888]
+; CHECK-GI-NEXT: mov v5.b[4], w11
+; CHECK-GI-NEXT: mov v2.b[5], w10
+; CHECK-GI-NEXT: mov v3.b[5], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #128]
+; CHECK-GI-NEXT: ldr w11, [sp, #1016]
+; CHECK-GI-NEXT: ldr w10, [sp, #256]
+; CHECK-GI-NEXT: mov v0.b[6], w6
+; CHECK-GI-NEXT: mov v4.b[5], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #768]
+; CHECK-GI-NEXT: mov v1.b[6], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #896]
; CHECK-GI-NEXT: mov v5.b[5], w11
-; CHECK-GI-NEXT: mov v6.b[5], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1024]
-; CHECK-GI-NEXT: mov v7.b[5], w12
-; CHECK-GI-NEXT: ldr w11, [sp, #968]
+; CHECK-GI-NEXT: mov v2.b[6], w10
+; CHECK-GI-NEXT: mov v3.b[6], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #136]
+; CHECK-GI-NEXT: ldr w11, [sp, #1024]
+; CHECK-GI-NEXT: ldr w10, [sp, #264]
+; CHECK-GI-NEXT: mov v0.b[7], w7
+; CHECK-GI-NEXT: mov v4.b[6], w8
+; CHECK-GI-NEXT: mov v1.b[7], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #904]
+; CHECK-GI-NEXT: mov v5.b[6], w11
+; CHECK-GI-NEXT: mov v2.b[7], w10
+; CHECK-GI-NEXT: ldr w8, [sp, #16]
+; CHECK-GI-NEXT: mov v3.b[7], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #144]
; CHECK-GI-NEXT: ldr w12, [sp, #1032]
-; CHECK-GI-NEXT: mov v16.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1136]
-; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
-; CHECK-GI-NEXT: mov v4.b[7], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #1144]
-; CHECK-GI-NEXT: mov v5.b[6], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1168]
-; CHECK-GI-NEXT: mov v6.b[6], w10
-; CHECK-GI-NEXT: mov v7.b[6], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1232]
-; CHECK-GI-NEXT: ldr w10, [sp, #1176]
-; CHECK-GI-NEXT: mov v16.b[4], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1240]
-; CHECK-GI-NEXT: fmov s17, w14
-; CHECK-GI-NEXT: fmov s18, w9
-; CHECK-GI-NEXT: ldr w14, [sp, #1096]
-; CHECK-GI-NEXT: ldr w9, [sp, #1160]
-; CHECK-GI-NEXT: mov v5.b[7], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1184]
-; CHECK-GI-NEXT: mov v6.b[7], w12
-; CHECK-GI-NEXT: mov v17.b[1], w10
-; CHECK-GI-NEXT: ldr w12, [sp, #1248]
-; CHECK-GI-NEXT: mov v7.b[7], w14
-; CHECK-GI-NEXT: mov v18.b[1], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1296]
-; CHECK-GI-NEXT: mov v16.b[5], w15
-; CHECK-GI-NEXT: ldr w15, [sp, #1360]
-; CHECK-GI-NEXT: ldr w14, [sp, #1304]
-; CHECK-GI-NEXT: ldr w10, [sp, #1152]
-; CHECK-GI-NEXT: fmov s19, w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1424]
-; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0
-; CHECK-GI-NEXT: mov v17.b[2], w11
-; CHECK-GI-NEXT: fmov s20, w15
-; CHECK-GI-NEXT: ldr w11, [sp, #1368]
-; CHECK-GI-NEXT: fmov s21, w13
-; CHECK-GI-NEXT: mov v18.b[2], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1432]
-; CHECK-GI-NEXT: mov v19.b[1], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1192]
-; CHECK-GI-NEXT: ldr w13, [sp, #1200]
-; CHECK-GI-NEXT: mov v20.b[1], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1312]
-; CHECK-GI-NEXT: mov v16.b[6], w10
-; CHECK-GI-NEXT: mov v21.b[1], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1256]
-; CHECK-GI-NEXT: mov v17.b[3], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1440]
-; CHECK-GI-NEXT: ldr w10, [sp, #1376]
-; CHECK-GI-NEXT: ushll v22.4s, v3.4h, #0
-; CHECK-GI-NEXT: mov v18.b[3], w12
-; CHECK-GI-NEXT: mov v19.b[2], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1208]
-; CHECK-GI-NEXT: mov v20.b[2], w10
-; CHECK-GI-NEXT: ldr w10, [sp, #1320]
-; CHECK-GI-NEXT: mov v16.b[7], w9
-; CHECK-GI-NEXT: mov v21.b[2], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1264]
-; CHECK-GI-NEXT: mov v17.b[4], w13
-; CHECK-GI-NEXT: ldr w9, [sp, #1384]
-; CHECK-GI-NEXT: ldr w13, [sp, #1448]
-; CHECK-GI-NEXT: ldr w12, [sp, #1216]
-; CHECK-GI-NEXT: mov v18.b[4], w14
-; CHECK-GI-NEXT: mov v19.b[3], w10
-; CHECK-GI-NEXT: ldr w14, [sp, #1272]
-; CHECK-GI-NEXT: mov v20.b[3], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1328]
-; CHECK-GI-NEXT: ldr w10, [sp, #1224]
-; CHECK-GI-NEXT: mov v17.b[5], w11
-; CHECK-GI-NEXT: mov v21.b[3], w13
-; CHECK-GI-NEXT: ldr w13, [sp, #1280]
-; CHECK-GI-NEXT: ldr w11, [sp, #1392]
-; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0
-; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT: mov v18.b[5], w14
-; CHECK-GI-NEXT: mov v19.b[4], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1336]
-; CHECK-GI-NEXT: ldr w14, [sp, #1456]
-; CHECK-GI-NEXT: mov v20.b[4], w11
-; CHECK-GI-NEXT: ldr w11, [sp, #1288]
-; CHECK-GI-NEXT: mov v17.b[6], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1344]
-; CHECK-GI-NEXT: sshll v27.4s, v16.4h, #0
-; CHECK-GI-NEXT: mov v21.b[4], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1400]
-; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0
-; CHECK-GI-NEXT: mov v18.b[6], w13
-; CHECK-GI-NEXT: mov v19.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1464]
-; CHECK-GI-NEXT: mov v20.b[5], w14
-; CHECK-GI-NEXT: ldr w14, [sp, #1352]
-; CHECK-GI-NEXT: ldr w13, [sp, #1416]
-; CHECK-GI-NEXT: mov v17.b[7], w10
-; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
-; CHECK-GI-NEXT: mov v21.b[5], w9
-; CHECK-GI-NEXT: ldr w9, [sp, #1408]
-; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: mov v18.b[7], w11
-; CHECK-GI-NEXT: mov v19.b[6], w12
-; CHECK-GI-NEXT: ldr w12, [sp, #1472]
-; CHECK-GI-NEXT: mov v20.b[6], w9
+; CHECK-GI-NEXT: mov v0.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #784]
+; CHECK-GI-NEXT: ldr w11, [sp, #272]
+; CHECK-GI-NEXT: mov v4.b[7], w9
+; CHECK-GI-NEXT: mov v1.b[8], w10
+; CHECK-GI-NEXT: ldr w10, [sp, #912]
+; CHECK-GI-NEXT: mov v5.b[7], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #24]
+; CHECK-GI-NEXT: ldr w12, [sp, #1040]
+; CHECK-GI-NEXT: mov v3.b[8], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #152]
+; CHECK-GI-NEXT: mov v2.b[8], w11
+; CHECK-GI-NEXT: mov v0.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #792]
+; CHECK-GI-NEXT: ldr w11, [sp, #280]
+; CHECK-GI-NEXT: mov v4.b[8], w10
+; CHECK-GI-NEXT: mov v1.b[9], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #920]
+; CHECK-GI-NEXT: mov v5.b[8], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #32]
+; CHECK-GI-NEXT: ldr w12, [sp, #1048]
+; CHECK-GI-NEXT: mov v3.b[9], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #160]
+; CHECK-GI-NEXT: mov v2.b[9], w11
+; CHECK-GI-NEXT: mov v0.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #800]
+; CHECK-GI-NEXT: ldr w11, [sp, #288]
+; CHECK-GI-NEXT: mov v4.b[9], w10
+; CHECK-GI-NEXT: mov v1.b[10], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #928]
+; CHECK-GI-NEXT: mov v5.b[9], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #40]
+; CHECK-GI-NEXT: ldr w12, [sp, #1056]
+; CHECK-GI-NEXT: mov v3.b[10], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #168]
+; CHECK-GI-NEXT: mov v2.b[10], w11
+; CHECK-GI-NEXT: mov v0.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #808]
+; CHECK-GI-NEXT: ldr w11, [sp, #296]
+; CHECK-GI-NEXT: mov v4.b[10], w10
+; CHECK-GI-NEXT: mov v1.b[11], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #936]
+; CHECK-GI-NEXT: mov v5.b[10], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #48]
+; CHECK-GI-NEXT: ldr w12, [sp, #1064]
+; CHECK-GI-NEXT: mov v3.b[11], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #176]
+; CHECK-GI-NEXT: mov v2.b[11], w11
+; CHECK-GI-NEXT: mov v0.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #816]
+; CHECK-GI-NEXT: ldr w11, [sp, #304]
+; CHECK-GI-NEXT: mov v4.b[11], w10
+; CHECK-GI-NEXT: mov v1.b[12], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #944]
+; CHECK-GI-NEXT: mov v5.b[11], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #56]
+; CHECK-GI-NEXT: ldr w12, [sp, #1072]
+; CHECK-GI-NEXT: mov v3.b[12], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #184]
+; CHECK-GI-NEXT: mov v2.b[12], w11
+; CHECK-GI-NEXT: mov v0.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #824]
+; CHECK-GI-NEXT: ldr w11, [sp, #312]
+; CHECK-GI-NEXT: mov v4.b[12], w10
+; CHECK-GI-NEXT: mov v1.b[13], w8
+; CHECK-GI-NEXT: ldr w10, [sp, #952]
+; CHECK-GI-NEXT: mov v5.b[12], w12
+; CHECK-GI-NEXT: ldr w8, [sp, #64]
+; CHECK-GI-NEXT: ldr w12, [sp, #1080]
+; CHECK-GI-NEXT: mov v3.b[13], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #192]
+; CHECK-GI-NEXT: mov v2.b[13], w11
+; CHECK-GI-NEXT: mov v0.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #832]
+; CHECK-GI-NEXT: ldr w11, [sp, #320]
+; CHECK-GI-NEXT: mov v4.b[13], w10
+; CHECK-GI-NEXT: mov v1.b[14], w9
+; CHECK-GI-NEXT: ldr w10, [sp, #960]
+; CHECK-GI-NEXT: mov v5.b[13], w12
+; CHECK-GI-NEXT: ldr w9, [sp, #72]
+; CHECK-GI-NEXT: ldr w12, [sp, #1088]
+; CHECK-GI-NEXT: mov v3.b[14], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #200]
+; CHECK-GI-NEXT: mov v2.b[14], w11
+; CHECK-GI-NEXT: mov v0.b[15], w9
+; CHECK-GI-NEXT: ldr w9, [sp, #840]
+; CHECK-GI-NEXT: ldr w11, [sp, #328]
+; CHECK-GI-NEXT: mov v4.b[14], w10
+; CHECK-GI-NEXT: mov v1.b[15], w8
+; CHECK-GI-NEXT: ldr w8, [sp, #968]
+; CHECK-GI-NEXT: mov v5.b[14], w12
+; CHECK-GI-NEXT: ldr w10, [sp, #1096]
+; CHECK-GI-NEXT: mov v3.b[15], w9
+; CHECK-GI-NEXT: mov v2.b[15], w11
+; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b
+; CHECK-GI-NEXT: mov v4.b[15], w8
+; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b
+; CHECK-GI-NEXT: mov v5.b[15], w10
+; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b
+; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b
+; CHECK-GI-NEXT: addv s0, v7.4s
+; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b
+; CHECK-GI-NEXT: addv s1, v16.4s
+; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b
+; CHECK-GI-NEXT: addv s2, v17.4s
+; CHECK-GI-NEXT: addv s4, v20.4s
+; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: ushll v1.4s, v2.4h, #0
-; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0
-; CHECK-GI-NEXT: ldr w15, [sp, #1480]
-; CHECK-GI-NEXT: mov v21.b[6], w12
-; CHECK-GI-NEXT: ushll v24.4s, v5.4h, #0
-; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0
-; CHECK-GI-NEXT: sshll v18.8h, v18.8b, #0
-; CHECK-GI-NEXT: mov v19.b[7], w14
-; CHECK-GI-NEXT: mul v1.4s, v1.4s, v27.4s
-; CHECK-GI-NEXT: sshll v28.4s, v17.4h, #0
-; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0
-; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s
-; CHECK-GI-NEXT: mov v20.b[7], w13
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v25.4s, v6.4h, #0
-; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0
-; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0
-; CHECK-GI-NEXT: mov v21.b[7], w15
-; CHECK-GI-NEXT: sshll v19.8h, v19.8b, #0
-; CHECK-GI-NEXT: mul v16.4s, v22.4s, v28.4s
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v17.4s
-; CHECK-GI-NEXT: addv s1, v1.4s
-; CHECK-GI-NEXT: addv s2, v2.4s
-; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0
-; CHECK-GI-NEXT: mul v17.4s, v23.4s, v29.4s
-; CHECK-GI-NEXT: sshll v20.8h, v20.8b, #0
-; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s
-; CHECK-GI-NEXT: sshll v30.4s, v19.4h, #0
-; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0
-; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
-; CHECK-GI-NEXT: addv s16, v16.4s
-; CHECK-GI-NEXT: addv s3, v3.4s
-; CHECK-GI-NEXT: fmov w10, s1
-; CHECK-GI-NEXT: fmov w11, s2
-; CHECK-GI-NEXT: sshll v31.4s, v20.4h, #0
-; CHECK-GI-NEXT: sshll v21.8h, v21.8b, #0
-; CHECK-GI-NEXT: addv s17, v17.4s
-; CHECK-GI-NEXT: mul v18.4s, v24.4s, v30.4s
-; CHECK-GI-NEXT: mul v5.4s, v5.4s, v19.4s
-; CHECK-GI-NEXT: addv s4, v4.4s
-; CHECK-GI-NEXT: fmov w12, s3
-; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: fmov w11, s16
-; CHECK-GI-NEXT: mul v19.4s, v25.4s, v31.4s
-; CHECK-GI-NEXT: fmov w13, s17
-; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0
-; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
-; CHECK-GI-NEXT: addv s18, v18.4s
-; CHECK-GI-NEXT: addv s5, v5.4s
-; CHECK-GI-NEXT: sshll v27.4s, v21.4h, #0
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: sshll2 v21.4s, v21.8h, #0
-; CHECK-GI-NEXT: mul v6.4s, v6.4s, v20.4s
-; CHECK-GI-NEXT: add w11, w12, w13
-; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: addv s19, v19.4s
-; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mul v20.4s, v26.4s, v27.4s
-; CHECK-GI-NEXT: fmov w14, s0
-; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s18
-; CHECK-GI-NEXT: addv s6, v6.4s
-; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: add w9, w9, w14
-; CHECK-GI-NEXT: addv s1, v20.4s
+; CHECK-GI-NEXT: addv s3, v18.4s
+; CHECK-GI-NEXT: addv s5, v19.4s
+; CHECK-GI-NEXT: fmov w10, s2
+; CHECK-GI-NEXT: add w8, w8, w9
+; CHECK-GI-NEXT: fmov w9, s4
+; CHECK-GI-NEXT: fmov w11, s3
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: add w12, w12, w13
-; CHECK-GI-NEXT: fmov w13, s19
-; CHECK-GI-NEXT: addv s2, v7.4s
-; CHECK-GI-NEXT: add w11, w12, w13
-; CHECK-GI-NEXT: fmov w12, s6
-; CHECK-GI-NEXT: fmov w13, s2
-; CHECK-GI-NEXT: add w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s1
; CHECK-GI-NEXT: add w10, w10, w11
-; CHECK-GI-NEXT: add w11, w12, w13
+; CHECK-GI-NEXT: fmov w11, s5
; CHECK-GI-NEXT: add w9, w10, w11
; CHECK-GI-NEXT: add w0, w8, w9
; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
- %az = zext <48 x i8> %a to <48 x i32>
- %bz = sext <48 x i8> %b to <48 x i32>
- %m1 = mul nuw nsw <48 x i32> %az, %bz
- %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)
- %cz = zext <48 x i8> %c to <48 x i32>
- %dz = sext <48 x i8> %d to <48 x i32>
- %m2 = mul nuw nsw <48 x i32> %cz, %dz
- %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)
+ %az = sext <48 x i8> %a to <48 x i32>
+ %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)
+ %cz = sext <48 x i8> %c to <48 x i32>
+ %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)
%x = add i32 %r1, %r2
ret i32 %x
}
More information about the llvm-commits
mailing list