[llvm] 338314f - [ARM] Lower v16i8 -> i64 VMLA reductions.
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 14 10:11:45 PDT 2021
Author: David Green
Date: 2021-07-14T18:11:32+01:00
New Revision: 338314f9c26d4594d49fdd3a7656d71c77255c54
URL: https://github.com/llvm/llvm-project/commit/338314f9c26d4594d49fdd3a7656d71c77255c54
DIFF: https://github.com/llvm/llvm-project/commit/338314f9c26d4594d49fdd3a7656d71c77255c54.diff
LOG: [ARM] Lower v16i8 -> i64 VMLA reductions.
MVE does not have a VMLALV instruction that can perform v16i8 -> i64
reductions, like it does for v8i16->i64 and v4i32->i64 reductions. That
means that the pattern to create them will be spilt up by type
legalization, creating a lot of instructions.
This extends the patterns for matching i64 reductions a little to handle
the v16i8->i64 case. We need to turn them into a pair of v8i16->i64
VMLALVs that each perform half of the reduction and are summed together
(so the later is a VMLALVA). The order of the lanes does not matter for
the reduction so we generate a MVEEXT for the extension, that will
either be folded into a extending load or can be optimized to a
VREV/VMOVL. Some of the resulting codegen isn't optimal, but will be
improved in a later patch.
Differential Revision: https://reviews.llvm.org/D105680
Added:
Modified:
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 2317448ace59c..5894afbb578eb 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -15954,12 +15954,12 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
SDLoc dl(N);
// We are looking for something that will have illegal types if left alone,
- // but that we can convert to a single instruction undef MVE. For example
+ // but that we can convert to a single instruction under MVE. For example
// vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
// or
// vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
- // Cases:
+ // The legal cases are:
// VADDV u/s 8/16/32
// VMLAV u/s 8/16/32
// VADDLV u/s 32
@@ -16067,6 +16067,32 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return false;
};
auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
+ // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64
+ // reductions. The operands are extended with MVEEXT, but as they are
+ // reductions the lane orders do not matter. MVEEXT may be combined with
+ // loads to produce two extending loads, or else they will be expanded to
+ // VREV/VMOVL.
+ EVT VT = Ops[0].getValueType();
+ if (VT == MVT::v16i8) {
+ assert((Opcode == ARMISD::VMLALVs || Opcode == ARMISD::VMLALVu) &&
+ "Unexpected illegal long reduction opcode");
+ bool IsUnsigned = Opcode == ARMISD::VMLALVu;
+
+ SDValue Ext0 =
+ DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
+ DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
+ SDValue Ext1 =
+ DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
+ DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
+
+ SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
+ Ext0, Ext1);
+ SDValue MLA1 =
+ DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
+ DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1),
+ Ext0.getValue(1), Ext1.getValue(1));
+ return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1));
+ }
SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
SDValue(Node.getNode(), 1));
@@ -16113,10 +16139,10 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND,
- {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
+ {MVT::v16i8, MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
return Create64bitNode(ARMISD::VMLALVs, {A, B});
if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND,
- {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
+ {MVT::v16i8, MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
return Create64bitNode(ARMISD::VMLALVu, {A, B});
if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
index a6e6a9cc81bae..44222e58cc9bf 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
@@ -532,136 +532,19 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: add_v16i8_v16i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vmov.u8 r0, q1[1]
-; CHECK-NEXT: vmov.u8 r1, q1[0]
-; CHECK-NEXT: vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT: vmov.u8 r1, q0[1]
-; CHECK-NEXT: vmov.u8 r2, q0[0]
-; CHECK-NEXT: vmov.i64 q2, #0xff
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r1
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r0, s12
-; CHECK-NEXT: vmov r1, s16
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umull r0, r12, r1, r0
-; CHECK-NEXT: vmov.u8 r1, q0[2]
-; CHECK-NEXT: smlabb r0, r3, r2, r0
-; CHECK-NEXT: vmov.u8 r2, q1[3]
-; CHECK-NEXT: vmov.u8 r3, q1[2]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[3]
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r1, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umull r1, r2, r1, r2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r12, r2
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.u8 r2, q1[5]
-; CHECK-NEXT: vmov.u8 r3, q1[4]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[5]
-; CHECK-NEXT: vmov.u8 r2, q0[4]
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[7]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[6]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[7]
-; CHECK-NEXT: vmov.u8 r2, q0[6]
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[9]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[8]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[9]
-; CHECK-NEXT: vmov.u8 r2, q0[8]
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[11]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[10]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[11]
-; CHECK-NEXT: vmov.u8 r2, q0[10]
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[13]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[12]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[13]
-; CHECK-NEXT: vmov.u8 r2, q0[12]
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[15]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[14]
-; CHECK-NEXT: vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[15]
-; CHECK-NEXT: vmov.u8 r2, q0[14]
-; CHECK-NEXT: vand q1, q1, q2
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r3
-; CHECK-NEXT: vmov r12, s4
-; CHECK-NEXT: vand q0, q0, q2
-; CHECK-NEXT: vmov r2, s0
-; CHECK-NEXT: vmov r3, s2
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: .pad #32
+; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: add r2, sp, #16
+; CHECK-NEXT: mov r3, sp
+; CHECK-NEXT: vstrw.32 q1, [r2]
+; CHECK-NEXT: vstrw.32 q0, [r3]
+; CHECK-NEXT: vldrb.u16 q0, [r2]
+; CHECK-NEXT: vldrb.u16 q1, [r3]
+; CHECK-NEXT: vmlalv.u16 r0, r1, q1, q0
+; CHECK-NEXT: vldrb.u16 q0, [r2, #8]
+; CHECK-NEXT: vldrb.u16 q1, [r3, #8]
+; CHECK-NEXT: vmlalva.u16 r0, r1, q1, q0
+; CHECK-NEXT: add sp, #32
; CHECK-NEXT: bx lr
entry:
%xx = zext <16 x i8> %x to <16 x i64>
@@ -674,54 +557,19 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: add_v16i8_v16i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.s8 r0, q1[1]
-; CHECK-NEXT: vmov.s8 r1, q0[1]
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov.s8 r2, q1[0]
-; CHECK-NEXT: vmov.s8 r3, q0[0]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[2]
-; CHECK-NEXT: vmov.s8 r3, q0[2]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[3]
-; CHECK-NEXT: vmov.s8 r3, q0[3]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[4]
-; CHECK-NEXT: vmov.s8 r3, q0[4]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[5]
-; CHECK-NEXT: vmov.s8 r3, q0[5]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[6]
-; CHECK-NEXT: vmov.s8 r3, q0[6]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[7]
-; CHECK-NEXT: vmov.s8 r3, q0[7]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[8]
-; CHECK-NEXT: vmov.s8 r3, q0[8]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[9]
-; CHECK-NEXT: vmov.s8 r3, q0[9]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[10]
-; CHECK-NEXT: vmov.s8 r3, q0[10]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[11]
-; CHECK-NEXT: vmov.s8 r3, q0[11]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[12]
-; CHECK-NEXT: vmov.s8 r3, q0[12]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[13]
-; CHECK-NEXT: vmov.s8 r3, q0[13]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[14]
-; CHECK-NEXT: vmov.s8 r3, q0[14]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q1[15]
-; CHECK-NEXT: vmov.s8 r3, q0[15]
-; CHECK-NEXT: smlal r0, r1, r3, r2
+; CHECK-NEXT: .pad #32
+; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: add r2, sp, #16
+; CHECK-NEXT: mov r3, sp
+; CHECK-NEXT: vstrw.32 q1, [r2]
+; CHECK-NEXT: vstrw.32 q0, [r3]
+; CHECK-NEXT: vldrb.s16 q0, [r2]
+; CHECK-NEXT: vldrb.s16 q1, [r3]
+; CHECK-NEXT: vmlalv.s16 r0, r1, q1, q0
+; CHECK-NEXT: vldrb.s16 q0, [r2, #8]
+; CHECK-NEXT: vldrb.s16 q1, [r3, #8]
+; CHECK-NEXT: vmlalva.s16 r0, r1, q1, q0
+; CHECK-NEXT: add sp, #32
; CHECK-NEXT: bx lr
entry:
%xx = sext <16 x i8> %x to <16 x i64>
@@ -734,138 +582,14 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext_load(<16 x i8> *%xp, <16 x i8> *%yp) {
; CHECK-LABEL: add_v16i8_v16i64_zext_load:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vldrw.u32 q1, [r1]
-; CHECK-NEXT: vmov.i64 q0, #0xff
-; CHECK-NEXT: vmov.u8 r1, q1[1]
-; CHECK-NEXT: vmov.u8 r2, q1[0]
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: vand q3, q2, q0
-; CHECK-NEXT: vldrw.u32 q2, [r0]
-; CHECK-NEXT: vmov r1, s12
-; CHECK-NEXT: vmov.u8 r0, q2[1]
-; CHECK-NEXT: vmov.u8 r2, q2[0]
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r0
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r0, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umull r0, r12, r0, r1
-; CHECK-NEXT: vmov.u8 r1, q2[2]
-; CHECK-NEXT: smlabb r0, r3, r2, r0
-; CHECK-NEXT: vmov.u8 r2, q1[3]
-; CHECK-NEXT: vmov.u8 r3, q1[2]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[3]
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r1, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umull r1, r2, r1, r2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r12, r2
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.u8 r2, q1[5]
-; CHECK-NEXT: vmov.u8 r3, q1[4]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[5]
-; CHECK-NEXT: vmov.u8 r2, q2[4]
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[7]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[6]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[7]
-; CHECK-NEXT: vmov.u8 r2, q2[6]
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[9]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[8]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[9]
-; CHECK-NEXT: vmov.u8 r2, q2[8]
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[11]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[10]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[11]
-; CHECK-NEXT: vmov.u8 r2, q2[10]
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[13]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[12]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[13]
-; CHECK-NEXT: vmov.u8 r2, q2[12]
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r3, s18
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u8 r2, q1[15]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u8 r3, q1[14]
-; CHECK-NEXT: vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q2[15]
-; CHECK-NEXT: vmov.u8 r2, q2[14]
-; CHECK-NEXT: vand q1, q1, q0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r3
-; CHECK-NEXT: vmov r12, s4
-; CHECK-NEXT: vand q0, q2, q0
-; CHECK-NEXT: vmov r2, s0
-; CHECK-NEXT: vmov r3, s2
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: vldrb.u16 q0, [r1]
+; CHECK-NEXT: vldrb.u16 q1, [r0]
+; CHECK-NEXT: vmlalv.u16 r2, r3, q1, q0
+; CHECK-NEXT: vldrb.u16 q0, [r1, #8]
+; CHECK-NEXT: vldrb.u16 q1, [r0, #8]
+; CHECK-NEXT: vmlalva.u16 r2, r3, q1, q0
+; CHECK-NEXT: mov r0, r2
+; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: bx lr
entry:
%x = load <16 x i8>, <16 x i8>* %xp
@@ -880,56 +604,14 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext_load(<16 x i8> *%xp, <16 x i8> *%yp) {
; CHECK-LABEL: add_v16i8_v16i64_sext_load:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: vldrw.u32 q1, [r0]
-; CHECK-NEXT: vmov.s8 r1, q0[1]
-; CHECK-NEXT: vmov.s8 r0, q1[1]
-; CHECK-NEXT: smull r0, r1, r0, r1
-; CHECK-NEXT: vmov.s8 r2, q0[0]
-; CHECK-NEXT: vmov.s8 r3, q1[0]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[2]
-; CHECK-NEXT: vmov.s8 r3, q1[2]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[3]
-; CHECK-NEXT: vmov.s8 r3, q1[3]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[4]
-; CHECK-NEXT: vmov.s8 r3, q1[4]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[5]
-; CHECK-NEXT: vmov.s8 r3, q1[5]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[6]
-; CHECK-NEXT: vmov.s8 r3, q1[6]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[7]
-; CHECK-NEXT: vmov.s8 r3, q1[7]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[8]
-; CHECK-NEXT: vmov.s8 r3, q1[8]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[9]
-; CHECK-NEXT: vmov.s8 r3, q1[9]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[10]
-; CHECK-NEXT: vmov.s8 r3, q1[10]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[11]
-; CHECK-NEXT: vmov.s8 r3, q1[11]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[12]
-; CHECK-NEXT: vmov.s8 r3, q1[12]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[13]
-; CHECK-NEXT: vmov.s8 r3, q1[13]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[14]
-; CHECK-NEXT: vmov.s8 r3, q1[14]
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.s8 r2, q0[15]
-; CHECK-NEXT: vmov.s8 r3, q1[15]
-; CHECK-NEXT: smlal r0, r1, r3, r2
+; CHECK-NEXT: vldrb.s16 q0, [r1]
+; CHECK-NEXT: vldrb.s16 q1, [r0]
+; CHECK-NEXT: vmlalv.s16 r2, r3, q1, q0
+; CHECK-NEXT: vldrb.s16 q0, [r1, #8]
+; CHECK-NEXT: vldrb.s16 q1, [r0, #8]
+; CHECK-NEXT: vmlalva.s16 r2, r3, q1, q0
+; CHECK-NEXT: mov r0, r2
+; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: bx lr
entry:
%x = load <16 x i8>, <16 x i8>* %xp
@@ -1542,141 +1224,24 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y, i64 %a) {
; CHECK-LABEL: add_v16i8_v16i64_acc_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vmov.u8 r2, q1[1]
-; CHECK-NEXT: vmov.u8 r3, q1[0]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u8 r3, q0[1]
-; CHECK-NEXT: vmov.u8 r2, q0[0]
-; CHECK-NEXT: vmov.i64 q2, #0xff
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r3
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov.u8 r4, q0[2]
-; CHECK-NEXT: vmov r3, s14
-; CHECK-NEXT: umull lr, r12, r2, r12
-; CHECK-NEXT: vmov r2, s18
-; CHECK-NEXT: smlabb lr, r2, r3, lr
-; CHECK-NEXT: vmov.u8 r3, q1[3]
-; CHECK-NEXT: vmov.u8 r2, q1[2]
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r3
-; CHECK-NEXT: vmov.u8 r3, q0[3]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, r3
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r3, s16
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: adds.w r2, r2, lr
-; CHECK-NEXT: vmov.u8 lr, q0[5]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov.u8 r12, q1[5]
-; CHECK-NEXT: vmov.u8 r4, q1[4]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[4]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q0[7]
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: vmov.u8 r4, q1[6]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov.u8 r12, q1[7]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[6]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q0[9]
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: vmov.u8 r4, q1[8]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov.u8 r12, q1[9]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[8]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q0[11]
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: vmov.u8 r4, q1[10]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov.u8 r12, q1[11]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[10]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q0[13]
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: vmov.u8 r4, q1[12]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov.u8 r12, q1[13]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[12]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q0[15]
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: vmov.u8 r4, q1[14]
-; CHECK-NEXT: adc.w r3, r3, r12
-; CHECK-NEXT: vmov.u8 r12, q1[15]
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q0[14]
-; CHECK-NEXT: vmov q0[2], q0[0], r4, lr
-; CHECK-NEXT: vand q1, q1, q2
-; CHECK-NEXT: vand q0, q0, q2
-; CHECK-NEXT: vmov r12, s4
-; CHECK-NEXT: vmov r4, s0
-; CHECK-NEXT: umlal r2, r3, r4, r12
-; CHECK-NEXT: vmov r12, s6
-; CHECK-NEXT: vmov r4, s2
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r2, r2, r4
-; CHECK-NEXT: adc.w r3, r3, r12
+; CHECK-NEXT: .save {r4, r5, r7, lr}
+; CHECK-NEXT: push {r4, r5, r7, lr}
+; CHECK-NEXT: .pad #32
+; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: add r4, sp, #16
+; CHECK-NEXT: mov r3, sp
+; CHECK-NEXT: vstrw.32 q1, [r4]
+; CHECK-NEXT: vstrw.32 q0, [r3]
+; CHECK-NEXT: vldrb.u16 q0, [r4]
+; CHECK-NEXT: vldrb.u16 q1, [r3]
+; CHECK-NEXT: vmlalv.u16 r2, r5, q1, q0
+; CHECK-NEXT: vldrb.u16 q0, [r4, #8]
+; CHECK-NEXT: vldrb.u16 q1, [r3, #8]
+; CHECK-NEXT: vmlalva.u16 r2, r5, q1, q0
; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: adcs r1, r5
+; CHECK-NEXT: add sp, #32
+; CHECK-NEXT: pop {r4, r5, r7, pc}
entry:
%xx = zext <16 x i8> %x to <16 x i64>
%yy = zext <16 x i8> %y to <16 x i64>
@@ -1689,59 +1254,24 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %y, i64 %a) {
; CHECK-LABEL: add_v16i8_v16i64_acc_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vmov.s8 r2, q1[1]
-; CHECK-NEXT: vmov.s8 r3, q0[1]
-; CHECK-NEXT: smull r3, lr, r3, r2
-; CHECK-NEXT: vmov.s8 r12, q1[0]
-; CHECK-NEXT: vmov.s8 r2, q0[0]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[2]
-; CHECK-NEXT: vmov.s8 r2, q0[2]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[3]
-; CHECK-NEXT: vmov.s8 r2, q0[3]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[4]
-; CHECK-NEXT: vmov.s8 r2, q0[4]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[5]
-; CHECK-NEXT: vmov.s8 r2, q0[5]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[6]
-; CHECK-NEXT: vmov.s8 r2, q0[6]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[7]
-; CHECK-NEXT: vmov.s8 r2, q0[7]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[8]
-; CHECK-NEXT: vmov.s8 r2, q0[8]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[9]
-; CHECK-NEXT: vmov.s8 r2, q0[9]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[10]
-; CHECK-NEXT: vmov.s8 r2, q0[10]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[11]
-; CHECK-NEXT: vmov.s8 r2, q0[11]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[12]
-; CHECK-NEXT: vmov.s8 r2, q0[12]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[13]
-; CHECK-NEXT: vmov.s8 r2, q0[13]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[14]
-; CHECK-NEXT: vmov.s8 r2, q0[14]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: vmov.s8 r12, q1[15]
-; CHECK-NEXT: vmov.s8 r2, q0[15]
-; CHECK-NEXT: smlal r3, lr, r2, r12
-; CHECK-NEXT: adds r0, r0, r3
-; CHECK-NEXT: adc.w r1, r1, lr
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: .save {r4, r5, r7, lr}
+; CHECK-NEXT: push {r4, r5, r7, lr}
+; CHECK-NEXT: .pad #32
+; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: add r4, sp, #16
+; CHECK-NEXT: mov r3, sp
+; CHECK-NEXT: vstrw.32 q1, [r4]
+; CHECK-NEXT: vstrw.32 q0, [r3]
+; CHECK-NEXT: vldrb.s16 q0, [r4]
+; CHECK-NEXT: vldrb.s16 q1, [r3]
+; CHECK-NEXT: vmlalv.s16 r2, r5, q1, q0
+; CHECK-NEXT: vldrb.s16 q0, [r4, #8]
+; CHECK-NEXT: vldrb.s16 q1, [r3, #8]
+; CHECK-NEXT: vmlalva.s16 r2, r5, q1, q0
+; CHECK-NEXT: adds r0, r0, r2
+; CHECK-NEXT: adcs r1, r5
+; CHECK-NEXT: add sp, #32
+; CHECK-NEXT: pop {r4, r5, r7, pc}
entry:
%xx = sext <16 x i8> %x to <16 x i64>
%yy = sext <16 x i8> %y to <16 x i64>
@@ -1754,143 +1284,17 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext_load(<16 x i8> *%xp, <16 x i8> *%yp, i64 %a) {
; CHECK-LABEL: add_v16i8_v16i64_acc_zext_load:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vldrw.u32 q1, [r1]
-; CHECK-NEXT: vmov.i64 q0, #0xff
-; CHECK-NEXT: vmov.u8 r12, q1[1]
-; CHECK-NEXT: vmov.u8 r1, q1[0]
-; CHECK-NEXT: vmov q2[2], q2[0], r1, r12
-; CHECK-NEXT: vand q3, q2, q0
-; CHECK-NEXT: vldrw.u32 q2, [r0]
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov.u8 r0, q2[1]
-; CHECK-NEXT: vmov.u8 r1, q2[0]
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT: vmov r1, s14
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov.u8 r4, q2[2]
-; CHECK-NEXT: vmov r0, s16
-; CHECK-NEXT: umull lr, r12, r0, r12
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: smlabb lr, r0, r1, lr
-; CHECK-NEXT: vmov.u8 r1, q1[3]
-; CHECK-NEXT: vmov.u8 r0, q1[2]
-; CHECK-NEXT: vmov q3[2], q3[0], r0, r1
-; CHECK-NEXT: vmov.u8 r1, q2[3]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, r1
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r0, s12
-; CHECK-NEXT: vmov r1, s16
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r0, r1, r1, r0
-; CHECK-NEXT: adds.w r0, r0, lr
-; CHECK-NEXT: vmov.u8 lr, q2[5]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov.u8 r12, q1[5]
-; CHECK-NEXT: vmov.u8 r4, q1[4]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[4]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q2[7]
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: vmov.u8 r4, q1[6]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov.u8 r12, q1[7]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[6]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q2[9]
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: vmov.u8 r4, q1[8]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov.u8 r12, q1[9]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[8]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q2[11]
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: vmov.u8 r4, q1[10]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov.u8 r12, q1[11]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[10]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q2[13]
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: vmov.u8 r4, q1[12]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov.u8 r12, q1[13]
-; CHECK-NEXT: vmov q3[2], q3[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[12]
-; CHECK-NEXT: vmov q4[2], q4[0], r4, lr
-; CHECK-NEXT: vand q3, q3, q0
-; CHECK-NEXT: vand q4, q4, q0
-; CHECK-NEXT: vmov r12, s12
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov.u8 lr, q2[15]
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s14
-; CHECK-NEXT: vmov r4, s18
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: vmov.u8 r4, q1[14]
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: vmov.u8 r12, q1[15]
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r12
-; CHECK-NEXT: vmov.u8 r4, q2[14]
-; CHECK-NEXT: vmov q2[2], q2[0], r4, lr
-; CHECK-NEXT: vand q1, q1, q0
-; CHECK-NEXT: vand q0, q2, q0
-; CHECK-NEXT: vmov r12, s4
-; CHECK-NEXT: vmov r4, s0
-; CHECK-NEXT: umlal r0, r1, r4, r12
-; CHECK-NEXT: vmov r12, s6
-; CHECK-NEXT: vmov r4, s2
-; CHECK-NEXT: umull r4, r12, r4, r12
-; CHECK-NEXT: adds r0, r0, r4
-; CHECK-NEXT: adc.w r1, r1, r12
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: .save {r5, lr}
+; CHECK-NEXT: push {r5, lr}
+; CHECK-NEXT: vldrb.u16 q0, [r1]
+; CHECK-NEXT: vldrb.u16 q1, [r0]
+; CHECK-NEXT: vmlalv.u16 r12, r5, q1, q0
+; CHECK-NEXT: vldrb.u16 q0, [r1, #8]
+; CHECK-NEXT: vldrb.u16 q1, [r0, #8]
+; CHECK-NEXT: vmlalva.u16 r12, r5, q1, q0
+; CHECK-NEXT: adds.w r0, r12, r2
+; CHECK-NEXT: adc.w r1, r5, r3
+; CHECK-NEXT: pop {r5, pc}
entry:
%x = load <16 x i8>, <16 x i8>* %xp
%y = load <16 x i8>, <16 x i8>* %yp
@@ -1905,61 +1309,17 @@ entry:
define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext_load(<16 x i8> *%xp, <16 x i8> *%yp, i64 %a) {
; CHECK-LABEL: add_v16i8_v16i64_acc_sext_load:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: vldrw.u32 q1, [r0]
-; CHECK-NEXT: vmov.s8 r1, q0[1]
-; CHECK-NEXT: vmov.s8 r0, q1[1]
-; CHECK-NEXT: smull r0, lr, r0, r1
-; CHECK-NEXT: vmov.s8 r12, q0[0]
-; CHECK-NEXT: vmov.s8 r1, q1[0]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[2]
-; CHECK-NEXT: vmov.s8 r1, q1[2]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[3]
-; CHECK-NEXT: vmov.s8 r1, q1[3]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[4]
-; CHECK-NEXT: vmov.s8 r1, q1[4]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[5]
-; CHECK-NEXT: vmov.s8 r1, q1[5]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[6]
-; CHECK-NEXT: vmov.s8 r1, q1[6]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[7]
-; CHECK-NEXT: vmov.s8 r1, q1[7]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[8]
-; CHECK-NEXT: vmov.s8 r1, q1[8]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[9]
-; CHECK-NEXT: vmov.s8 r1, q1[9]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[10]
-; CHECK-NEXT: vmov.s8 r1, q1[10]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[11]
-; CHECK-NEXT: vmov.s8 r1, q1[11]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[12]
-; CHECK-NEXT: vmov.s8 r1, q1[12]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[13]
-; CHECK-NEXT: vmov.s8 r1, q1[13]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[14]
-; CHECK-NEXT: vmov.s8 r1, q1[14]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: vmov.s8 r12, q0[15]
-; CHECK-NEXT: vmov.s8 r1, q1[15]
-; CHECK-NEXT: smlal r0, lr, r1, r12
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adc.w r1, lr, r3
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: .save {r5, lr}
+; CHECK-NEXT: push {r5, lr}
+; CHECK-NEXT: vldrb.s16 q0, [r1]
+; CHECK-NEXT: vldrb.s16 q1, [r0]
+; CHECK-NEXT: vmlalv.s16 r12, r5, q1, q0
+; CHECK-NEXT: vldrb.s16 q0, [r1, #8]
+; CHECK-NEXT: vldrb.s16 q1, [r0, #8]
+; CHECK-NEXT: vmlalva.s16 r12, r5, q1, q0
+; CHECK-NEXT: adds.w r0, r12, r2
+; CHECK-NEXT: adc.w r1, r5, r3
+; CHECK-NEXT: pop {r5, pc}
entry:
%x = load <16 x i8>, <16 x i8>* %xp
%y = load <16 x i8>, <16 x i8>* %yp
More information about the llvm-commits
mailing list