[llvm-branch-commits] [llvm] 901cc9b - [ARM] Extend lowering for i64 reductions
David Green via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jan 4 04:48:55 PST 2021
Author: David Green
Date: 2021-01-04T12:44:43Z
New Revision: 901cc9b6f30f120f2fbdc01f9eec3708c512186b
URL: https://github.com/llvm/llvm-project/commit/901cc9b6f30f120f2fbdc01f9eec3708c512186b
DIFF: https://github.com/llvm/llvm-project/commit/901cc9b6f30f120f2fbdc01f9eec3708c512186b.diff
LOG: [ARM] Extend lowering for i64 reductions
The lowering of a <4 x i16> or <4 x i8> vecreduce.add into an i64 would
previously be expanded, due to the i64 not being legal. This patch
adjusts our reduction matchers, making it produce a VADDLV(sext A to
v4i32) instead.
Differential Revision: https://reviews.llvm.org/D93622
Added:
Modified:
llvm/lib/Target/ARM/ARMISelLowering.cpp
llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 6eb1bdffdac4..6a8355f0c3e8 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -14959,12 +14959,23 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
// VADDLV u/s 32
// VMLALV u/s 16/32
+ // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
+ // extend it and use v4i32 instead.
+ auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
+ EVT AVT = A.getValueType();
+ if (!AVT.is128BitVector())
+ A = DAG.getNode(ExtendCode, dl,
+ AVT.changeVectorElementType(MVT::getIntegerVT(
+ 128 / AVT.getVectorMinNumElements())),
+ A);
+ return A;
+ };
auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
return SDValue();
SDValue A = N0->getOperand(0);
if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
- return A;
+ return ExtendIfNeeded(A, ExtendCode);
return SDValue();
};
auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
@@ -14978,7 +14989,7 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return SDValue();
SDValue A = Ext->getOperand(0);
if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
- return A;
+ return ExtendIfNeeded(A, ExtendCode);
return SDValue();
};
auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
@@ -15007,8 +15018,12 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
A = ExtA->getOperand(0);
B = ExtB->getOperand(0);
if (A.getValueType() == B.getValueType() &&
- llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
+ llvm::any_of(ExtTypes,
+ [&A](MVT Ty) { return A.getValueType() == Ty; })) {
+ A = ExtendIfNeeded(A, ExtendCode);
+ B = ExtendIfNeeded(B, ExtendCode);
return true;
+ }
return false;
};
auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
@@ -15037,8 +15052,12 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
A = ExtA->getOperand(0);
B = ExtB->getOperand(0);
if (A.getValueType() == B.getValueType() &&
- llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
+ llvm::any_of(ExtTypes,
+ [&A](MVT Ty) { return A.getValueType() == Ty; })) {
+ A = ExtendIfNeeded(A, ExtendCode);
+ B = ExtendIfNeeded(B, ExtendCode);
return true;
+ }
return false;
};
auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
@@ -15051,9 +15070,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A);
- if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}))
+ if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND,
+ {MVT::v4i8, MVT::v4i16, MVT::v4i32}))
return Create64bitNode(ARMISD::VADDLVs, {A});
- if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
+ if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND,
+ {MVT::v4i8, MVT::v4i16, MVT::v4i32}))
return Create64bitNode(ARMISD::VADDLVu, {A});
if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
@@ -15067,9 +15088,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
- if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
+ if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND,
+ {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask))
return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
- if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
+ if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND,
+ {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask))
return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
@@ -15083,9 +15106,11 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
- if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B))
+ if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND,
+ {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
return Create64bitNode(ARMISD::VMLALVs, {A, B});
- if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B))
+ if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND,
+ {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
return Create64bitNode(ARMISD::VMLALVu, {A, B});
if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
@@ -15098,9 +15123,13 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask))
return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
- if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, Mask))
+ if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND,
+ {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A,
+ B, Mask))
return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
- if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, Mask))
+ if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND,
+ {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A,
+ B, Mask))
return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
index f882582bf148..49bd421d17aa 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
@@ -251,31 +251,8 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x) {
; CHECK-LABEL: add_v4i16_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.f32 s4, s0
-; CHECK-NEXT: vmov.f32 s6, s1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s6
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: sxth r0, r0
-; CHECK-NEXT: vmov q1[2], q1[0], r1, r0
-; CHECK-NEXT: asrs r2, r0, #31
-; CHECK-NEXT: asrs r1, r1, #31
-; CHECK-NEXT: vmov q1[3], q1[1], r1, r2
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r1, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: adc.w r0, r1, r0, asr #31
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: adds r2, r2, r1
-; CHECK-NEXT: adc.w r1, r0, r1, asr #31
-; CHECK-NEXT: vmov r0, s6
-; CHECK-NEXT: sxth r3, r0
-; CHECK-NEXT: adds r0, r2, r3
-; CHECK-NEXT: adc.w r1, r1, r3, asr #31
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vaddlv.s32 r0, r1, q0
; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i16> %x to <4 x i64>
@@ -813,31 +790,9 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x) {
; CHECK-LABEL: add_v4i8_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.f32 s4, s0
-; CHECK-NEXT: vmov.f32 s6, s1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s6
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: vmov q1[2], q1[0], r1, r0
-; CHECK-NEXT: asrs r2, r0, #31
-; CHECK-NEXT: asrs r1, r1, #31
-; CHECK-NEXT: vmov q1[3], q1[1], r1, r2
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r1, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: adc.w r0, r1, r0, asr #31
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: adds r2, r2, r1
-; CHECK-NEXT: adc.w r1, r0, r1, asr #31
-; CHECK-NEXT: vmov r0, s6
-; CHECK-NEXT: sxtb r3, r0
-; CHECK-NEXT: adds r0, r2, r3
-; CHECK-NEXT: adc.w r1, r1, r3, asr #31
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vaddlv.s32 r0, r1, q0
; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i8> %x to <4 x i64>
@@ -1173,36 +1128,9 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_acc_sext(<4 x i16> %x, i64 %a) {
; CHECK-LABEL: add_v4i16_v4i64_acc_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vmov.f32 s4, s0
-; CHECK-NEXT: vmov.f32 s6, s1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT: asr.w r12, r2, #31
-; CHECK-NEXT: asrs r3, r3, #31
-; CHECK-NEXT: vmov q1[3], q1[1], r3, r12
-; CHECK-NEXT: vmov lr, s6
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r12, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds.w r3, r3, lr
-; CHECK-NEXT: adc.w r12, r12, r2, asr #31
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: adds r3, r3, r2
-; CHECK-NEXT: adc.w r12, r12, r2, asr #31
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: adds r3, r3, r2
-; CHECK-NEXT: adc.w r2, r12, r2, asr #31
-; CHECK-NEXT: adds r0, r0, r3
-; CHECK-NEXT: adcs r1, r2
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vaddlva.s32 r0, r1, q0
+; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i16> %x to <4 x i64>
%z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
@@ -1778,36 +1706,10 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_acc_sext(<4 x i8> %x, i64 %a) {
; CHECK-LABEL: add_v4i8_v4i64_acc_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vmov.f32 s4, s0
-; CHECK-NEXT: vmov.f32 s6, s1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: vmov q1[2], q1[0], r3, r2
-; CHECK-NEXT: asr.w r12, r2, #31
-; CHECK-NEXT: asrs r3, r3, #31
-; CHECK-NEXT: vmov q1[3], q1[1], r3, r12
-; CHECK-NEXT: vmov lr, s6
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: vmov r12, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds.w r3, r3, lr
-; CHECK-NEXT: adc.w r12, r12, r2, asr #31
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: adds r3, r3, r2
-; CHECK-NEXT: adc.w r12, r12, r2, asr #31
-; CHECK-NEXT: vmov r2, s6
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: adds r3, r3, r2
-; CHECK-NEXT: adc.w r2, r12, r2, asr #31
-; CHECK-NEXT: adds r0, r0, r3
-; CHECK-NEXT: adcs r1, r2
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vaddlva.s32 r0, r1, q0
+; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i8> %x to <4 x i64>
%z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
index a5ad4eff87e0..dc120b98961b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
@@ -421,45 +421,10 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %b) {
; CHECK-LABEL: add_v4i16_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.u16 q1, q1
; CHECK-NEXT: vmovlb.u16 q0, q0
-; CHECK-NEXT: vcmp.i32 eq, q1, zr
-; CHECK-NEXT: vmov.f32 s12, s0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov.f32 s14, s1
-; CHECK-NEXT: vmov.i64 q2, #0xffffffff
-; CHECK-NEXT: vand q3, q3, q2
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r2, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r2, r1
-; CHECK-NEXT: vand q1, q3, q1
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: vmov r12, s7
-; CHECK-NEXT: vmov r2, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vand q0, q1, q2
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r3
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r3
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
+; CHECK-NEXT: vmovlb.u16 q1, q1
+; CHECK-NEXT: vpt.i32 eq, q1, zr
+; CHECK-NEXT: vaddlvt.u32 r0, r1, q0
; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i16> %b, zeroinitializer
@@ -472,60 +437,11 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %b) {
; CHECK-LABEL: add_v4i16_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vmov.f32 s8, s0
+; CHECK-NEXT: vmovlb.s16 q0, q0
; CHECK-NEXT: vmovlb.u16 q1, q1
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vcmp.i32 eq, q1, zr
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: sxth r0, r0
-; CHECK-NEXT: vmov q2[2], q2[0], r1, r0
-; CHECK-NEXT: asrs r0, r0, #31
-; CHECK-NEXT: asrs r1, r1, #31
-; CHECK-NEXT: vmov q2[3], q2[1], r1, r0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r2, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r2, r1
-; CHECK-NEXT: vand q1, q2, q1
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: vmov r12, s7
-; CHECK-NEXT: vmov r2, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds.w lr, r1, r3
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: vmov q0[2], q0[0], r1, r3
-; CHECK-NEXT: asrs r3, r3, #31
-; CHECK-NEXT: asrs r1, r1, #31
-; CHECK-NEXT: vmov q0[3], q0[1], r1, r3
-; CHECK-NEXT: ubfx r1, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r1
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: vpt.i32 eq, q1, zr
+; CHECK-NEXT: vaddlvt.s32 r0, r1, q0
+; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i16> %b, zeroinitializer
%xx = sext <4 x i16> %x to <4 x i64>
@@ -1600,49 +1516,11 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %b) {
; CHECK-LABEL: add_v4i8_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vmov.i32 q3, #0xff
-; CHECK-NEXT: vmov.i64 q2, #0xffffffff
-; CHECK-NEXT: vand q1, q1, q3
-; CHECK-NEXT: vand q0, q0, q3
-; CHECK-NEXT: vcmp.i32 eq, q1, zr
-; CHECK-NEXT: vmov.f32 s16, s0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov.f32 s18, s1
-; CHECK-NEXT: vand q4, q4, q2
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r2, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r2, r1
-; CHECK-NEXT: vand q1, q4, q1
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: vmov r12, s7
-; CHECK-NEXT: vmov r2, s5
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vand q0, q1, q2
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r3
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r3
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: vmov.i32 q2, #0xff
+; CHECK-NEXT: vand q0, q0, q2
+; CHECK-NEXT: vand q1, q1, q2
+; CHECK-NEXT: vpt.i32 eq, q1, zr
+; CHECK-NEXT: vaddlvt.u32 r0, r1, q0
; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i8> %b, zeroinitializer
@@ -1655,58 +1533,12 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %b) {
; CHECK-LABEL: add_v4i8_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
+; CHECK-NEXT: vmovlb.s8 q0, q0
; CHECK-NEXT: vmov.i32 q2, #0xff
; CHECK-NEXT: vand q1, q1, q2
-; CHECK-NEXT: vmov.f32 s8, s0
-; CHECK-NEXT: vcmp.i32 eq, q1, zr
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r2, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r2, r1
-; CHECK-NEXT: vmov r1, s10
-; CHECK-NEXT: vmov r2, s0
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: asrs r1, r1, #31
-; CHECK-NEXT: asrs r2, r2, #31
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r1
-; CHECK-NEXT: vand q1, q2, q1
-; CHECK-NEXT: vmov.f32 s8, s2
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: vmov r1, s4
-; CHECK-NEXT: vmov.f32 s10, s3
-; CHECK-NEXT: vmov r12, s7
-; CHECK-NEXT: vmov r2, s5
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r3
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r3
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov r3, s8
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: vmov q0[2], q0[0], r3, r0
-; CHECK-NEXT: asrs r0, r0, #31
-; CHECK-NEXT: asrs r3, r3, #31
-; CHECK-NEXT: vmov q0[3], q0[1], r3, r0
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vpt.i32 eq, q1, zr
+; CHECK-NEXT: vaddlvt.s32 r0, r1, q0
; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i8> %b, zeroinitializer
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
index ba2651e43ea5..82ef43d96fe3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
@@ -173,35 +173,9 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: add_v4i16_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: vmovlb.u16 q1, q1
; CHECK-NEXT: vmovlb.u16 q0, q0
-; CHECK-NEXT: vmov.f32 s8, s4
-; CHECK-NEXT: vmov.f32 s12, s0
-; CHECK-NEXT: vmov.f32 s10, s5
-; CHECK-NEXT: vmov.f32 s14, s1
-; CHECK-NEXT: vmullb.u32 q4, q3, q2
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov r2, s18
-; CHECK-NEXT: vmov r3, s16
-; CHECK-NEXT: vmov r0, s19
-; CHECK-NEXT: vmov r1, s17
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmullb.u32 q0, q1, q2
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: adcs r0, r1
-; CHECK-NEXT: vmov r1, s1
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r1, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%xx = zext <4 x i16> %x to <4 x i64>
@@ -214,41 +188,9 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: add_v4i16_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.f32 s8, s4
-; CHECK-NEXT: vmov.f32 s10, s5
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov.f32 s8, s0
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vmov r1, s10
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: sxth r0, r0
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r0
-; CHECK-NEXT: vmov q2[3], q2[1], r3, r1
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov r3, s8
-; CHECK-NEXT: vmov r2, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds r0, r0, r3
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: adcs r1, r2
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: smlal r0, r1, r3, r2
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i16> %x to <4 x i64>
@@ -948,96 +890,10 @@ entry:
define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: add_v8i8_v8i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vmovlb.u8 q2, q1
+; CHECK-NEXT: vmovlb.u8 q1, q1
; CHECK-NEXT: vmovlb.u8 q0, q0
-; CHECK-NEXT: vmov.u16 r0, q2[1]
-; CHECK-NEXT: vmov.u16 r1, q2[0]
-; CHECK-NEXT: vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT: vmov.u16 r1, q0[1]
-; CHECK-NEXT: vmov.u16 r2, q0[0]
-; CHECK-NEXT: vmov.i64 q1, #0xffff
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r1
-; CHECK-NEXT: vand q3, q3, q1
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vmov r0, s14
-; CHECK-NEXT: vmov r1, s18
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r3, s16
-; CHECK-NEXT: umull r0, r1, r1, r0
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r0
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r1
-; CHECK-NEXT: vmov r2, s14
-; CHECK-NEXT: vmov r3, s12
-; CHECK-NEXT: vmov r0, s13
-; CHECK-NEXT: adds.w lr, r3, r2
-; CHECK-NEXT: vmov.u16 r3, q2[2]
-; CHECK-NEXT: adc.w r12, r0, r1
-; CHECK-NEXT: vmov.u16 r1, q2[3]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r1
-; CHECK-NEXT: vmov.u16 r3, q0[3]
-; CHECK-NEXT: vmov.u16 r0, q0[2]
-; CHECK-NEXT: vand q3, q3, q1
-; CHECK-NEXT: vmov q4[2], q4[0], r0, r3
-; CHECK-NEXT: vmov r1, s14
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vmov r3, s12
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: umull r0, r1, r0, r1
-; CHECK-NEXT: umull r2, r3, r2, r3
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r0
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r1
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r0, s13
-; CHECK-NEXT: vmov r3, s14
-; CHECK-NEXT: adds.w r2, r2, lr
-; CHECK-NEXT: adc.w r0, r0, r12
-; CHECK-NEXT: adds.w lr, r2, r3
-; CHECK-NEXT: vmov.u16 r3, q2[4]
-; CHECK-NEXT: adc.w r12, r0, r1
-; CHECK-NEXT: vmov.u16 r1, q2[5]
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r1
-; CHECK-NEXT: vmov.u16 r3, q0[5]
-; CHECK-NEXT: vmov.u16 r0, q0[4]
-; CHECK-NEXT: vand q3, q3, q1
-; CHECK-NEXT: vmov q4[2], q4[0], r0, r3
-; CHECK-NEXT: vmov r1, s14
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vmov r3, s12
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: umull r0, r1, r0, r1
-; CHECK-NEXT: umull r2, r3, r2, r3
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r0
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r1
-; CHECK-NEXT: vmov r2, s12
-; CHECK-NEXT: vmov r0, s13
-; CHECK-NEXT: adds.w r2, r2, lr
-; CHECK-NEXT: adc.w r3, r12, r0
-; CHECK-NEXT: vmov r0, s14
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: vmov.u16 r2, q2[7]
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vmov.u16 r3, q2[6]
-; CHECK-NEXT: vmov q2[2], q2[0], r3, r2
-; CHECK-NEXT: vmov.u16 r3, q0[7]
-; CHECK-NEXT: vmov.u16 r2, q0[6]
-; CHECK-NEXT: vand q2, q2, q1
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r3
-; CHECK-NEXT: vmov r12, s8
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r2, s0
-; CHECK-NEXT: vmov r3, s2
-; CHECK-NEXT: umlal r0, r1, r2, r12
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: umlal r0, r1, r3, r2
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: vmlalv.u16 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%xx = zext <8 x i8> %x to <8 x i64>
%yy = zext <8 x i8> %y to <8 x i64>
@@ -1049,74 +905,10 @@ entry:
define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: add_v8i8_v8i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r7, lr}
-; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: vmov.u16 r0, q1[1]
-; CHECK-NEXT: vmov.u16 r1, q0[1]
-; CHECK-NEXT: vmov.u16 r2, q1[0]
-; CHECK-NEXT: vmov.u16 r3, q0[0]
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r0
-; CHECK-NEXT: vmov q2[3], q2[1], r3, r1
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: vmov r3, s8
-; CHECK-NEXT: vmov r0, s9
-; CHECK-NEXT: adds.w lr, r3, r2
-; CHECK-NEXT: vmov.u16 r3, q0[3]
-; CHECK-NEXT: adc.w r12, r0, r1
-; CHECK-NEXT: vmov.u16 r1, q1[3]
-; CHECK-NEXT: vmov.u16 r0, q1[2]
-; CHECK-NEXT: vmov.u16 r2, q0[2]
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: smull r1, r3, r3, r1
-; CHECK-NEXT: smull r0, r2, r2, r0
-; CHECK-NEXT: vmov q2[2], q2[0], r0, r1
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r3
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r0, s9
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r0, r0, r12
-; CHECK-NEXT: adds.w lr, r1, r2
-; CHECK-NEXT: vmov.u16 r2, q1[5]
-; CHECK-NEXT: adc.w r12, r0, r3
-; CHECK-NEXT: vmov.u16 r3, q0[5]
-; CHECK-NEXT: vmov.u16 r0, q1[4]
-; CHECK-NEXT: vmov.u16 r1, q0[4]
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov q2[2], q2[0], r0, r2
-; CHECK-NEXT: vmov q2[3], q2[1], r1, r3
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r0, s9
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vmov.u16 r2, q1[6]
-; CHECK-NEXT: vmov.u16 r3, q0[6]
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov.u16 r2, q1[7]
-; CHECK-NEXT: vmov.u16 r3, q0[7]
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: vmovlb.s8 q1, q1
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%xx = sext <8 x i8> %x to <8 x i64>
%yy = sext <8 x i8> %y to <8 x i64>
@@ -1128,36 +920,10 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) {
; CHECK-LABEL: add_v4i8_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: vmov.i32 q2, #0xff
; CHECK-NEXT: vand q1, q1, q2
; CHECK-NEXT: vand q0, q0, q2
-; CHECK-NEXT: vmov.f32 s12, s4
-; CHECK-NEXT: vmov.f32 s8, s0
-; CHECK-NEXT: vmov.f32 s14, s5
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vmullb.u32 q4, q2, q3
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov r2, s18
-; CHECK-NEXT: vmov r3, s16
-; CHECK-NEXT: vmov r0, s19
-; CHECK-NEXT: vmov r1, s17
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmullb.u32 q0, q1, q2
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: adcs r0, r1
-; CHECK-NEXT: vmov r1, s1
-; CHECK-NEXT: adds r2, r2, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r1, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r2
-; CHECK-NEXT: adcs r1, r3
-; CHECK-NEXT: vpop {d8, d9}
+; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%xx = zext <4 x i8> %x to <4 x i64>
@@ -1170,41 +936,11 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) {
; CHECK-LABEL: add_v4i8_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmov.f32 s8, s4
-; CHECK-NEXT: vmov.f32 s10, s5
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov.f32 s8, s0
-; CHECK-NEXT: vmov.f32 s10, s1
-; CHECK-NEXT: vmov r1, s10
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r0
-; CHECK-NEXT: vmov q2[3], q2[1], r3, r1
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: vmov r3, s8
-; CHECK-NEXT: vmov r2, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: adds r0, r0, r3
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: adcs r1, r2
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: smlal r0, r1, r3, r2
-; CHECK-NEXT: vmov r2, s10
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smlal r0, r1, r3, r2
+; CHECK-NEXT: vmovlb.s8 q1, q1
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%xx = sext <4 x i8> %x to <4 x i64>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
index e4252019c3d8..f20dc9480ce1 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
@@ -285,52 +285,11 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {
; CHECK-LABEL: add_v4i16_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9, d10, d11}
-; CHECK-NEXT: vpush {d8, d9, d10, d11}
-; CHECK-NEXT: vmovlb.u16 q2, q2
; CHECK-NEXT: vmovlb.u16 q1, q1
-; CHECK-NEXT: vcmp.i32 eq, q2, zr
; CHECK-NEXT: vmovlb.u16 q0, q0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov.f32 s12, s4
-; CHECK-NEXT: vmov.f32 s16, s0
-; CHECK-NEXT: vmov.f32 s14, s5
-; CHECK-NEXT: vmov.f32 s18, s1
-; CHECK-NEXT: vmullb.u32 q5, q4, q3
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r1
-; CHECK-NEXT: vand q2, q5, q2
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r12, s11
-; CHECK-NEXT: vmov r2, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmullb.u32 q0, q1, q2
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r3
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r3
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9, d10, d11}
+; CHECK-NEXT: vmovlb.u16 q2, q2
+; CHECK-NEXT: vpt.i32 eq, q2, zr
+; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i16> %b, zeroinitializer
@@ -345,72 +304,12 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {
; CHECK-LABEL: add_v4i16_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: vmov.f32 s12, s4
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmovlb.s16 q0, q0
; CHECK-NEXT: vmovlb.u16 q2, q2
-; CHECK-NEXT: vmov.f32 s14, s5
-; CHECK-NEXT: vcmp.i32 eq, q2, zr
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s14
-; CHECK-NEXT: vmov.f32 s12, s0
-; CHECK-NEXT: vmov.f32 s14, s1
-; CHECK-NEXT: vmov r1, s14
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: sxth r0, r0
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r1
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r1
-; CHECK-NEXT: vand q2, q3, q2
-; CHECK-NEXT: vmov r1, s10
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: vmov r12, s11
-; CHECK-NEXT: vmov r3, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmov r4, s4
-; CHECK-NEXT: adds.w lr, r2, r1
-; CHECK-NEXT: vmov r1, s6
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: adc.w r12, r12, r3
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: sxth r4, r4
-; CHECK-NEXT: sxth r1, r1
-; CHECK-NEXT: sxth r2, r2
-; CHECK-NEXT: sxth r3, r3
-; CHECK-NEXT: smull r2, r4, r4, r2
-; CHECK-NEXT: smull r1, r3, r1, r3
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r1
-; CHECK-NEXT: ubfx r1, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r1
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: vpt.i32 eq, q2, zr
+; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i16> %b, zeroinitializer
%xx = sext <4 x i16> %x to <4 x i64>
@@ -1526,158 +1425,12 @@ entry:
define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {
; CHECK-LABEL: add_v8i8_v8i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9, d10, d11}
-; CHECK-NEXT: vpush {d8, d9, d10, d11}
-; CHECK-NEXT: vmovlb.u8 q3, q1
+; CHECK-NEXT: vmovlb.u8 q1, q1
; CHECK-NEXT: vmovlb.u8 q0, q0
-; CHECK-NEXT: vmov.u16 r0, q3[1]
-; CHECK-NEXT: vmov.u16 r1, q3[0]
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT: vmov.u16 r1, q0[1]
-; CHECK-NEXT: vmov.u16 r2, q0[0]
-; CHECK-NEXT: vmov.i64 q1, #0xffff
-; CHECK-NEXT: vmov q5[2], q5[0], r2, r1
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vand q5, q5, q1
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: vmov r1, s22
; CHECK-NEXT: vmovlb.u8 q2, q2
-; CHECK-NEXT: vmov r3, s20
-; CHECK-NEXT: vcmp.i16 eq, q2, zr
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov.i8 q2, #0x0
-; CHECK-NEXT: vmov.i8 q5, #0xff
-; CHECK-NEXT: vpsel q2, q5, q2
-; CHECK-NEXT: umull r0, r1, r1, r0
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r0
-; CHECK-NEXT: vmov.u16 r0, q2[2]
-; CHECK-NEXT: vmov q4[3], q4[1], r3, r1
-; CHECK-NEXT: vmov.u16 r1, q2[0]
-; CHECK-NEXT: vmov q5[2], q5[0], r1, r0
-; CHECK-NEXT: vmov.u16 r0, q2[3]
-; CHECK-NEXT: vmov.u16 r1, q2[1]
-; CHECK-NEXT: vmov q5[3], q5[1], r1, r0
-; CHECK-NEXT: vcmp.i32 ne, q5, zr
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q5[2], q5[0], r2, r1
-; CHECK-NEXT: vmov q5[3], q5[1], r2, r1
-; CHECK-NEXT: vand q4, q4, q5
-; CHECK-NEXT: vmov r1, s18
-; CHECK-NEXT: vmov r2, s16
-; CHECK-NEXT: vmov r12, s19
-; CHECK-NEXT: vmov r3, s17
-; CHECK-NEXT: adds.w lr, r2, r1
-; CHECK-NEXT: vmov.u16 r1, q3[2]
-; CHECK-NEXT: vmov.u16 r2, q0[2]
-; CHECK-NEXT: adc.w r12, r12, r3
-; CHECK-NEXT: vmov.u16 r3, q3[3]
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r3
-; CHECK-NEXT: vmov.u16 r3, q0[3]
-; CHECK-NEXT: vmov q5[2], q5[0], r2, r3
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vand q5, q5, q1
-; CHECK-NEXT: vmov r1, s18
-; CHECK-NEXT: vmov r2, s22
-; CHECK-NEXT: vmov r3, s16
-; CHECK-NEXT: vmov r4, s20
-; CHECK-NEXT: umull r1, r2, r2, r1
-; CHECK-NEXT: umull r3, r4, r4, r3
-; CHECK-NEXT: vmov q4[2], q4[0], r3, r1
-; CHECK-NEXT: ubfx r1, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: vmov q4[3], q4[1], r4, r2
-; CHECK-NEXT: vmov q5[2], q5[0], r0, r1
-; CHECK-NEXT: vmov.u16 r4, q0[4]
-; CHECK-NEXT: vmov q5[3], q5[1], r0, r1
-; CHECK-NEXT: vand q4, q4, q5
-; CHECK-NEXT: vmov r1, s16
-; CHECK-NEXT: vmov r0, s17
-; CHECK-NEXT: vmov r3, s19
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s18
-; CHECK-NEXT: adds.w r12, r1, r0
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vmov.u16 r2, q3[5]
-; CHECK-NEXT: vmov.u16 r3, q3[4]
-; CHECK-NEXT: vmov q4[2], q4[0], r3, r2
-; CHECK-NEXT: vmov.u16 r3, q0[5]
-; CHECK-NEXT: vmov q5[2], q5[0], r4, r3
-; CHECK-NEXT: vand q4, q4, q1
-; CHECK-NEXT: vand q5, q5, q1
-; CHECK-NEXT: vmov r2, s18
-; CHECK-NEXT: vmov r3, s22
-; CHECK-NEXT: vmov r4, s16
-; CHECK-NEXT: vmov r0, s20
-; CHECK-NEXT: umull r2, r3, r3, r2
-; CHECK-NEXT: umull r0, r4, r0, r4
-; CHECK-NEXT: vmov q4[2], q4[0], r0, r2
-; CHECK-NEXT: vmov.u16 r0, q2[6]
-; CHECK-NEXT: vmov.u16 r2, q2[4]
-; CHECK-NEXT: vmov q4[3], q4[1], r4, r3
-; CHECK-NEXT: vmov q5[2], q5[0], r2, r0
-; CHECK-NEXT: vmov.u16 r0, q2[7]
-; CHECK-NEXT: vmov.u16 r2, q2[5]
-; CHECK-NEXT: vmov q5[3], q5[1], r2, r0
-; CHECK-NEXT: vcmp.i32 ne, q5, zr
-; CHECK-NEXT: vmrs r2, p0
-; CHECK-NEXT: and r3, r2, #1
-; CHECK-NEXT: ubfx r0, r2, #4, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r3, r0
-; CHECK-NEXT: vmov q2[3], q2[1], r3, r0
-; CHECK-NEXT: vand q2, q4, q2
-; CHECK-NEXT: vmov r3, s8
-; CHECK-NEXT: vmov r0, s9
-; CHECK-NEXT: vmov r4, s11
-; CHECK-NEXT: adds.w r3, r3, r12
-; CHECK-NEXT: adcs r1, r0
-; CHECK-NEXT: vmov r0, s10
-; CHECK-NEXT: adds.w r12, r3, r0
-; CHECK-NEXT: vmov.u16 r3, q3[7]
-; CHECK-NEXT: adc.w lr, r1, r4
-; CHECK-NEXT: vmov.u16 r4, q3[6]
-; CHECK-NEXT: vmov q2[2], q2[0], r4, r3
-; CHECK-NEXT: vmov.u16 r4, q0[7]
-; CHECK-NEXT: vmov.u16 r0, q0[6]
-; CHECK-NEXT: vand q2, q2, q1
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r4, s8
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: umull r0, r3, r0, r3
-; CHECK-NEXT: umull r1, r4, r1, r4
-; CHECK-NEXT: vmov q0[2], q0[0], r1, r0
-; CHECK-NEXT: ubfx r0, r2, #12, #1
-; CHECK-NEXT: ubfx r1, r2, #8, #1
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT: vmov q1[2], q1[0], r1, r0
-; CHECK-NEXT: vmov q1[3], q1[1], r1, r0
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adds.w r1, r1, r12
-; CHECK-NEXT: adc.w r2, lr, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9, d10, d11}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: vpt.i16 eq, q2, zr
+; CHECK-NEXT: vmlalvt.u16 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%c = icmp eq <8 x i8> %b, zeroinitializer
%xx = zext <8 x i8> %x to <8 x i64>
@@ -1691,139 +1444,12 @@ entry:
define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {
; CHECK-LABEL: add_v8i8_v8i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
+; CHECK-NEXT: vmovlb.s8 q1, q1
+; CHECK-NEXT: vmovlb.s8 q0, q0
; CHECK-NEXT: vmovlb.u8 q2, q2
-; CHECK-NEXT: vmov.i8 q3, #0xff
-; CHECK-NEXT: vcmp.i16 eq, q2, zr
-; CHECK-NEXT: vmov.i8 q2, #0x0
-; CHECK-NEXT: vpsel q2, q3, q2
-; CHECK-NEXT: vmov.u16 r3, q1[0]
-; CHECK-NEXT: vmov.u16 r0, q2[2]
-; CHECK-NEXT: vmov.u16 r1, q2[0]
-; CHECK-NEXT: vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT: vmov.u16 r0, q2[3]
-; CHECK-NEXT: vmov.u16 r1, q2[1]
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: vmov q3[3], q3[1], r1, r0
-; CHECK-NEXT: vmov.u16 r4, q0[4]
-; CHECK-NEXT: vcmp.i32 ne, q3, zr
-; CHECK-NEXT: sxtb r4, r4
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r1
-; CHECK-NEXT: vmov q3[3], q3[1], r2, r1
-; CHECK-NEXT: vmov.u16 r1, q1[1]
-; CHECK-NEXT: vmov.u16 r2, q0[1]
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: smull r1, r12, r2, r1
-; CHECK-NEXT: vmov.u16 r2, q0[0]
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: smull r2, r3, r2, r3
-; CHECK-NEXT: vmov q4[2], q4[0], r2, r1
-; CHECK-NEXT: vmov q4[3], q4[1], r3, r12
-; CHECK-NEXT: vand q3, q4, q3
-; CHECK-NEXT: vmov r3, s14
-; CHECK-NEXT: vmov r1, s12
-; CHECK-NEXT: vmov r12, s15
-; CHECK-NEXT: vmov r2, s13
-; CHECK-NEXT: adds.w lr, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: vmov.u16 r1, q1[2]
-; CHECK-NEXT: vmov q3[2], q3[0], r0, r3
-; CHECK-NEXT: adc.w r12, r12, r2
-; CHECK-NEXT: vmov q3[3], q3[1], r0, r3
-; CHECK-NEXT: vmov.u16 r0, q1[3]
-; CHECK-NEXT: vmov.u16 r3, q0[3]
-; CHECK-NEXT: vmov.u16 r2, q0[2]
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: smull r0, r3, r3, r0
-; CHECK-NEXT: smull r1, r2, r2, r1
-; CHECK-NEXT: vmov q4[2], q4[0], r1, r0
-; CHECK-NEXT: vmov q4[3], q4[1], r2, r3
-; CHECK-NEXT: vand q3, q4, q3
-; CHECK-NEXT: vmov r1, s12
-; CHECK-NEXT: vmov r0, s13
-; CHECK-NEXT: vmov r3, s15
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s14
-; CHECK-NEXT: adds.w r12, r1, r0
-; CHECK-NEXT: vmov.u16 r1, q1[4]
-; CHECK-NEXT: adc.w lr, r2, r3
-; CHECK-NEXT: vmov.u16 r2, q2[6]
-; CHECK-NEXT: vmov.u16 r3, q2[4]
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
-; CHECK-NEXT: vmov.u16 r2, q2[7]
-; CHECK-NEXT: vmov.u16 r3, q2[5]
-; CHECK-NEXT: smull r1, r4, r4, r1
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r2
-; CHECK-NEXT: vcmp.i32 ne, q3, zr
-; CHECK-NEXT: vmrs r2, p0
-; CHECK-NEXT: and r0, r2, #1
-; CHECK-NEXT: ubfx r3, r2, #4, #1
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r0, r3
-; CHECK-NEXT: vmov q2[3], q2[1], r0, r3
-; CHECK-NEXT: vmov.u16 r0, q1[5]
-; CHECK-NEXT: vmov.u16 r3, q0[5]
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r0, r3, r3, r0
-; CHECK-NEXT: vmov q3[2], q3[0], r1, r0
-; CHECK-NEXT: vmov q3[3], q3[1], r4, r3
-; CHECK-NEXT: vand q2, q3, q2
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r0, s9
-; CHECK-NEXT: vmov r4, s10
-; CHECK-NEXT: vmov r3, s11
-; CHECK-NEXT: adds.w r1, r1, r12
-; CHECK-NEXT: adc.w r0, r0, lr
-; CHECK-NEXT: adds r1, r1, r4
-; CHECK-NEXT: vmov.u16 r4, q1[6]
-; CHECK-NEXT: adc.w r12, r0, r3
-; CHECK-NEXT: ubfx r3, r2, #12, #1
-; CHECK-NEXT: ubfx r2, r2, #8, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: vmov.u16 r0, q0[6]
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r3
-; CHECK-NEXT: sxtb r4, r4
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r3
-; CHECK-NEXT: vmov.u16 r2, q1[7]
-; CHECK-NEXT: vmov.u16 r3, q0[7]
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: smull r0, r4, r0, r4
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r2
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT: vand q0, q0, q2
-; CHECK-NEXT: vmov r2, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adds r1, r1, r2
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: vpt.i16 eq, q2, zr
+; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%c = icmp eq <8 x i8> %b, zeroinitializer
%xx = sext <8 x i8> %x to <8 x i64>
@@ -1837,53 +1463,12 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {
; CHECK-LABEL: add_v4i8_v4i64_zext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT: vmov.i32 q3, #0xff
-; CHECK-NEXT: vand q2, q2, q3
; CHECK-NEXT: vand q1, q1, q3
-; CHECK-NEXT: vcmp.i32 eq, q2, zr
; CHECK-NEXT: vand q0, q0, q3
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov.f32 s16, s4
-; CHECK-NEXT: vmov.f32 s20, s0
-; CHECK-NEXT: vmov.f32 s18, s5
-; CHECK-NEXT: vmov.f32 s22, s1
-; CHECK-NEXT: vmullb.u32 q6, q5, q4
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r1
-; CHECK-NEXT: vand q2, q6, q2
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: vmov r1, s8
-; CHECK-NEXT: vmov r12, s11
-; CHECK-NEXT: vmov r2, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmullb.u32 q0, q1, q2
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: ubfx r3, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsb.w r3, r3, #0
-; CHECK-NEXT: rsb.w r0, r0, #0
-; CHECK-NEXT: adc.w r2, r2, r12
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r3
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r3
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: adds r1, r1, r3
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adcs r2, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: vand q2, q2, q3
+; CHECK-NEXT: vpt.i32 eq, q2, zr
+; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q1
; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i8> %b, zeroinitializer
@@ -1898,76 +1483,15 @@ entry:
define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {
; CHECK-LABEL: add_v4i8_v4i64_sext:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, lr}
-; CHECK-NEXT: push {r4, lr}
-; CHECK-NEXT: .vsave {d8, d9}
-; CHECK-NEXT: vpush {d8, d9}
-; CHECK-NEXT: vmov.f32 s12, s4
-; CHECK-NEXT: vmov.i32 q4, #0xff
-; CHECK-NEXT: vmov.f32 s14, s5
-; CHECK-NEXT: vand q2, q2, q4
-; CHECK-NEXT: vmov r2, s4
-; CHECK-NEXT: vcmp.i32 eq, q2, zr
-; CHECK-NEXT: vmov r3, s0
-; CHECK-NEXT: vmov r0, s14
-; CHECK-NEXT: vmov.f32 s12, s0
-; CHECK-NEXT: vmov.f32 s14, s1
-; CHECK-NEXT: vmov r1, s14
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r2, r3, r3, r2
-; CHECK-NEXT: sxtb r0, r0
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: smull r0, r1, r1, r0
-; CHECK-NEXT: vmov q3[2], q3[0], r2, r0
-; CHECK-NEXT: vmrs r0, p0
-; CHECK-NEXT: vmov q3[3], q3[1], r3, r1
-; CHECK-NEXT: and r2, r0, #1
-; CHECK-NEXT: ubfx r1, r0, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: vmov q2[2], q2[0], r2, r1
-; CHECK-NEXT: vmov q2[3], q2[1], r2, r1
-; CHECK-NEXT: vand q2, q3, q2
-; CHECK-NEXT: vmov r1, s10
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: vmov r12, s11
-; CHECK-NEXT: vmov r3, s9
-; CHECK-NEXT: vmov.f32 s8, s6
-; CHECK-NEXT: vmov.f32 s10, s7
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s6, s3
-; CHECK-NEXT: vmov r4, s4
-; CHECK-NEXT: adds.w lr, r2, r1
-; CHECK-NEXT: vmov r1, s6
-; CHECK-NEXT: vmov r2, s8
-; CHECK-NEXT: adc.w r12, r12, r3
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: sxtb r4, r4
-; CHECK-NEXT: sxtb r1, r1
-; CHECK-NEXT: sxtb r2, r2
-; CHECK-NEXT: sxtb r3, r3
-; CHECK-NEXT: smull r2, r4, r4, r2
-; CHECK-NEXT: smull r1, r3, r1, r3
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r1
-; CHECK-NEXT: ubfx r1, r0, #12, #1
-; CHECK-NEXT: ubfx r0, r0, #8, #1
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: rsbs r0, r0, #0
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r3
-; CHECK-NEXT: vmov q1[2], q1[0], r0, r1
-; CHECK-NEXT: vmov q1[3], q1[1], r0, r1
-; CHECK-NEXT: vand q0, q0, q1
-; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vmov r3, s3
-; CHECK-NEXT: adds.w r1, r1, lr
-; CHECK-NEXT: adc.w r2, r12, r0
-; CHECK-NEXT: vmov r0, s2
-; CHECK-NEXT: adds r0, r0, r1
-; CHECK-NEXT: adc.w r1, r2, r3
-; CHECK-NEXT: vpop {d8, d9}
-; CHECK-NEXT: pop {r4, pc}
+; CHECK-NEXT: vmovlb.s8 q1, q1
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmov.i32 q3, #0xff
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vand q2, q2, q3
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vpt.i32 eq, q2, zr
+; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q1
+; CHECK-NEXT: bx lr
entry:
%c = icmp eq <4 x i8> %b, zeroinitializer
%xx = sext <4 x i8> %x to <4 x i64>
More information about the llvm-branch-commits
mailing list