[llvm] 90f24be - [ARM] Fold And/Or into CSel if possible
David Green via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 9 05:29:03 PST 2023
Author: David Green
Date: 2023-01-09T13:28:57Z
New Revision: 90f24bef47227d58f2ccdcc481ca22eff32248ca
URL: https://github.com/llvm/llvm-project/commit/90f24bef47227d58f2ccdcc481ca22eff32248ca
DIFF: https://github.com/llvm/llvm-project/commit/90f24bef47227d58f2ccdcc481ca22eff32248ca.diff
LOG: [ARM] Fold And/Or into CSel if possible
This is the ARM equivalent of D141119, where we fold `and x, (csel 0, 1, cc)`
to `csel ZR, x, cc` if we know that x is 0/1 and for `or x, (csel 0, 1, cc)`
emit `csinc x, ZR, cc`. The or pattern gets recognized from a cmov under Arm.
Differential Revision: https://reviews.llvm.org/D141137
Added:
Modified:
llvm/lib/Target/ARM/ARMInstrInfo.td
llvm/lib/Target/ARM/ARMInstrThumb2.td
llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
llvm/test/CodeGen/Thumb2/csel-andor-onebit.ll
llvm/test/CodeGen/Thumb2/mve-nounrolledremainder.ll
llvm/test/CodeGen/Thumb2/mve-pred-and.ll
llvm/test/CodeGen/Thumb2/mve-pred-or.ll
llvm/test/CodeGen/Thumb2/mve-vcmp.ll
llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index 5c56e4714aff1..20d4db8410b46 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -426,6 +426,12 @@ def top16Zero: PatLeaf<(i32 GPR:$src), [{
CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(32, 16));
}]>;
+// topbitsallzero - Return true if all bits except the lowest bit are known zero
+def topbitsallzero32 : PatLeaf<(i32 GPRwithZR:$src), [{
+ return SDValue(N,0)->getValueType(0) == MVT::i32 &&
+ CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(32, 31));
+ }]>;
+
class BinOpFrag<dag res> : PatFrag<(ops node:$LHS, node:$RHS), res>;
class UnOpFrag <dag res> : PatFrag<(ops node:$Src), res>;
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index ebe4864032619..f06f9f1b99b57 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -5657,6 +5657,11 @@ let Predicates = [HasV8_1MMainline] in {
defm : ModifiedV8_1CSEL<t2CSINC, (add rGPR:$fval, 1)>;
defm : ModifiedV8_1CSEL<t2CSINV, (xor rGPR:$fval, -1)>;
defm : ModifiedV8_1CSEL<t2CSNEG, (sub 0, rGPR:$fval)>;
+
+ def : T2Pat<(ARMcmov (topbitsallzero32:$Rn), (i32 1), cmovpred:$imm),
+ (t2CSINC $Rn, ZR, $imm)>;
+ def : T2Pat<(and (topbitsallzero32:$Rn), (ARMcsinc (i32 0), (i32 0), cmovpred:$imm)),
+ (t2CSEL ZR, $Rn, $imm)>;
}
// CS aliases.
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index 27e26fd17016f..eff4aab225241 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -8,25 +8,25 @@ define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocaptur
; CHECK-NEXT: cmp r3, #0
; CHECK-NEXT: beq.w .LBB0_11
; CHECK-NEXT: @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r0, r3, lsl #2
; CHECK-NEXT: add.w r4, r2, r3, lsl #2
-; CHECK-NEXT: cmp r5, r2
-; CHECK-NEXT: cset r12, hi
+; CHECK-NEXT: add.w lr, r0, r3, lsl #2
; CHECK-NEXT: cmp r4, r0
+; CHECK-NEXT: cset r4, hi
+; CHECK-NEXT: cmp lr, r2
+; CHECK-NEXT: csel r12, zr, r4, ls
+; CHECK-NEXT: cmp lr, r1
+; CHECK-NEXT: add.w r4, r1, r3, lsl #2
; CHECK-NEXT: cset lr, hi
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r5, r1, r3, lsl #2
+; CHECK-NEXT: cmp r4, r0
; CHECK-NEXT: cset r4, hi
-; CHECK-NEXT: cmp r5, r0
-; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: tst r5, r4
+; CHECK-NEXT: tst.w r4, lr
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r5, lr, r12
+; CHECK-NEXT: cmpeq.w r12, #0
; CHECK-NEXT: beq .LBB0_4
; CHECK-NEXT: @ %bb.2: @ %for.body.preheader
-; CHECK-NEXT: subs r5, r3, #1
+; CHECK-NEXT: subs r4, r3, #1
; CHECK-NEXT: and r12, r3, #3
-; CHECK-NEXT: cmp r5, #3
+; CHECK-NEXT: cmp r4, #3
; CHECK-NEXT: bhs .LBB0_6
; CHECK-NEXT: @ %bb.3:
; CHECK-NEXT: movs r3, #0
@@ -43,10 +43,10 @@ define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocaptur
; CHECK-NEXT: b .LBB0_11
; CHECK-NEXT: .LBB0_6: @ %for.body.preheader.new
; CHECK-NEXT: bic r3, r3, #3
-; CHECK-NEXT: movs r5, #1
+; CHECK-NEXT: movs r4, #1
; CHECK-NEXT: subs r3, #4
+; CHECK-NEXT: add.w lr, r4, r3, lsr #2
; CHECK-NEXT: movs r4, #0
-; CHECK-NEXT: add.w lr, r5, r3, lsr #2
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: .LBB0_7: @ %for.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index 7a279eab36d9d..125240d8e2d21 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -14,21 +14,21 @@ define arm_aapcs_vfpcc void @float_float_mul(ptr nocapture readonly %a, ptr noca
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: b .LBB0_4
; CHECK-NEXT: .LBB0_3: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r2, r3, lsl #2
-; CHECK-NEXT: add.w r6, r1, r3, lsl #2
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r4, r0, r3, lsl #2
+; CHECK-NEXT: add.w r7, r1, r3, lsl #2
+; CHECK-NEXT: add.w r6, r2, r3, lsl #2
+; CHECK-NEXT: cmp r7, r2
+; CHECK-NEXT: add.w r5, r0, r3, lsl #2
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: cmp r6, r2
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
+; CHECK-NEXT: cmp r5, r2
; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: cmp r4, r2
-; CHECK-NEXT: cset r4, hi
; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: tst r4, r5
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r7, r6
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB0_11
; CHECK-NEXT: .LBB0_4: @ %for.body.preheader22
; CHECK-NEXT: mvn.w r7, r12
@@ -224,21 +224,21 @@ define arm_aapcs_vfpcc void @float_float_add(ptr nocapture readonly %a, ptr noca
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: b .LBB1_4
; CHECK-NEXT: .LBB1_3: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r2, r3, lsl #2
-; CHECK-NEXT: add.w r6, r1, r3, lsl #2
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r4, r0, r3, lsl #2
+; CHECK-NEXT: add.w r7, r1, r3, lsl #2
+; CHECK-NEXT: add.w r6, r2, r3, lsl #2
+; CHECK-NEXT: cmp r7, r2
+; CHECK-NEXT: add.w r5, r0, r3, lsl #2
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: cmp r6, r2
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
+; CHECK-NEXT: cmp r5, r2
; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: cmp r4, r2
-; CHECK-NEXT: cset r4, hi
; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: tst r4, r5
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r7, r6
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB1_11
; CHECK-NEXT: .LBB1_4: @ %for.body.preheader22
; CHECK-NEXT: mvn.w r7, r12
@@ -434,21 +434,21 @@ define arm_aapcs_vfpcc void @float_float_sub(ptr nocapture readonly %a, ptr noca
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: b .LBB2_4
; CHECK-NEXT: .LBB2_3: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r2, r3, lsl #2
-; CHECK-NEXT: add.w r6, r1, r3, lsl #2
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r4, r0, r3, lsl #2
+; CHECK-NEXT: add.w r7, r1, r3, lsl #2
+; CHECK-NEXT: add.w r6, r2, r3, lsl #2
+; CHECK-NEXT: cmp r7, r2
+; CHECK-NEXT: add.w r5, r0, r3, lsl #2
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: cmp r6, r2
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
+; CHECK-NEXT: cmp r5, r2
; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: cmp r4, r2
-; CHECK-NEXT: cset r4, hi
; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: tst r4, r5
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r7, r6
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB2_11
; CHECK-NEXT: .LBB2_4: @ %for.body.preheader22
; CHECK-NEXT: mvn.w r7, r12
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
index 5eb7bac51128b..d41d7d2c1a51d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
@@ -332,20 +332,20 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly
; CHECK-NEXT: cmp r4, #0
; CHECK-NEXT: beq.w .LBB5_11
; CHECK-NEXT: @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT: add.w r5, r3, r4, lsl #2
-; CHECK-NEXT: adds r6, r1, r4
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r7, r0, r4
-; CHECK-NEXT: cset r12, hi
-; CHECK-NEXT: cmp r6, r3
-; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
-; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: adds r7, r1, r4
+; CHECK-NEXT: add.w r6, r3, r4, lsl #2
; CHECK-NEXT: cmp r7, r3
+; CHECK-NEXT: add.w r5, r0, r4
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: tst r7, r5
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
+; CHECK-NEXT: cset r6, hi
+; CHECK-NEXT: cmp r5, r3
+; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r6, r12
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB5_4
; CHECK-NEXT: @ %bb.2: @ %for.body.preheader
; CHECK-NEXT: subs r7, r4, #1
@@ -608,20 +608,20 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonl
; CHECK-NEXT: cmp r4, #0
; CHECK-NEXT: beq.w .LBB7_11
; CHECK-NEXT: @ %bb.1: @ %for.body.lr.ph
-; CHECK-NEXT: add.w r5, r3, r4, lsl #2
-; CHECK-NEXT: adds r6, r1, r4
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r7, r0, r4
-; CHECK-NEXT: cset r12, hi
-; CHECK-NEXT: cmp r6, r3
-; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
-; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: adds r7, r1, r4
+; CHECK-NEXT: add.w r6, r3, r4, lsl #2
; CHECK-NEXT: cmp r7, r3
+; CHECK-NEXT: add.w r5, r0, r4
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: tst r7, r5
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
+; CHECK-NEXT: cset r6, hi
+; CHECK-NEXT: cmp r5, r3
+; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r6, r12
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB7_4
; CHECK-NEXT: @ %bb.2: @ %for.body.preheader
; CHECK-NEXT: subs r7, r4, #1
@@ -884,20 +884,20 @@ define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(i32* nocapture readonly
; CHECK-NEXT: cmp r4, #0
; CHECK-NEXT: beq.w .LBB9_11
; CHECK-NEXT: @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r3, r4, lsl #2
-; CHECK-NEXT: add.w r6, r1, r4, lsl #2
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r7, r0, r4, lsl #2
-; CHECK-NEXT: cset r12, hi
-; CHECK-NEXT: cmp r6, r3
-; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
-; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: add.w r7, r1, r4, lsl #2
+; CHECK-NEXT: add.w r6, r3, r4, lsl #2
; CHECK-NEXT: cmp r7, r3
+; CHECK-NEXT: add.w r5, r0, r4, lsl #2
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: tst r7, r5
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
+; CHECK-NEXT: cset r6, hi
+; CHECK-NEXT: cmp r5, r3
+; CHECK-NEXT: cset r5, hi
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r6, r12
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB9_4
; CHECK-NEXT: @ %bb.2: @ %for.body.preheader
; CHECK-NEXT: subs r7, r4, #1
diff --git a/llvm/test/CodeGen/Thumb2/csel-andor-onebit.ll b/llvm/test/CodeGen/Thumb2/csel-andor-onebit.ll
index 64f4de7258630..d9e18b5c0942e 100644
--- a/llvm/test/CodeGen/Thumb2/csel-andor-onebit.ll
+++ b/llvm/test/CodeGen/Thumb2/csel-andor-onebit.ll
@@ -6,8 +6,7 @@ define i32 @ori32i32_eq(i32 %x, i32 %y) {
; CHECK: @ %bb.0:
; CHECK-NEXT: and r0, r0, #1
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r0, #1
+; CHECK-NEXT: csinc r0, r0, zr, eq
; CHECK-NEXT: bx lr
%xa = and i32 %x, 1
%c = icmp eq i32 %y, 0
@@ -21,8 +20,7 @@ define i32 @ori32_eq_c(i32 %x, i32 %y) {
; CHECK: @ %bb.0:
; CHECK-NEXT: and r0, r0, #1
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r0, #1
+; CHECK-NEXT: csinc r0, r0, zr, eq
; CHECK-NEXT: bx lr
%xa = and i32 %x, 1
%c = icmp eq i32 %y, 0
@@ -34,10 +32,9 @@ define i32 @ori32_eq_c(i32 %x, i32 %y) {
define i32 @ori32i64_eq(i32 %x, i64 %y) {
; CHECK-LABEL: ori32i64_eq:
; CHECK: @ %bb.0:
-; CHECK-NEXT: and r0, r0, #1
; CHECK-NEXT: orrs.w r1, r2, r3
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r0, #1
+; CHECK-NEXT: and r0, r0, #1
+; CHECK-NEXT: csinc r0, r0, zr, eq
; CHECK-NEXT: bx lr
%xa = and i32 %x, 1
%c = icmp eq i64 %y, 0
@@ -51,8 +48,7 @@ define i32 @ori32_sgt(i32 %x, i32 %y) {
; CHECK: @ %bb.0:
; CHECK-NEXT: and r0, r0, #1
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: it gt
-; CHECK-NEXT: movgt r0, #1
+; CHECK-NEXT: csinc r0, r0, zr, gt
; CHECK-NEXT: bx lr
%xa = and i32 %x, 1
%c = icmp sgt i32 %y, 0
@@ -80,11 +76,10 @@ define i32 @ori32_toomanybits(i32 %x, i32 %y) {
define i32 @andi32_ne(i8 %x, i8 %y) {
; CHECK-LABEL: andi32_ne:
; CHECK: @ %bb.0:
-; CHECK-NEXT: tst.w r1, #255
-; CHECK-NEXT: cset r1, ne
; CHECK-NEXT: tst.w r0, #255
; CHECK-NEXT: cset r0, eq
-; CHECK-NEXT: ands r0, r1
+; CHECK-NEXT: tst.w r1, #255
+; CHECK-NEXT: csel r0, zr, r0, eq
; CHECK-NEXT: bx lr
%xc = icmp eq i8 %x, 0
%xa = zext i1 %xc to i32
@@ -101,8 +96,7 @@ define i32 @andi32_sgt(i8 %x, i8 %y) {
; CHECK-NEXT: sxtb r1, r1
; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, gt
-; CHECK-NEXT: ands r0, r1
+; CHECK-NEXT: csel r0, zr, r0, le
; CHECK-NEXT: bx lr
%xc = icmp eq i8 %x, 0
%xa = zext i1 %xc to i32
@@ -163,11 +157,10 @@ define i64 @ori64_eq_c(i64 %x, i32 %y) {
define i64 @andi64_ne(i8 %x, i8 %y) {
; CHECK-LABEL: andi64_ne:
; CHECK: @ %bb.0:
-; CHECK-NEXT: tst.w r1, #255
-; CHECK-NEXT: cset r1, ne
; CHECK-NEXT: tst.w r0, #255
; CHECK-NEXT: cset r0, eq
-; CHECK-NEXT: ands r0, r1
+; CHECK-NEXT: tst.w r1, #255
+; CHECK-NEXT: csel r0, zr, r0, eq
; CHECK-NEXT: movs r1, #0
; CHECK-NEXT: bx lr
%xc = icmp eq i8 %x, 0
diff --git a/llvm/test/CodeGen/Thumb2/mve-nounrolledremainder.ll b/llvm/test/CodeGen/Thumb2/mve-nounrolledremainder.ll
index 491c800c5c14b..d596e6c908cad 100644
--- a/llvm/test/CodeGen/Thumb2/mve-nounrolledremainder.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-nounrolledremainder.ll
@@ -4,25 +4,26 @@
define void @tailpred(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr nocapture %pDst, i32 %blockSize) {
; CHECK-LABEL: tailpred:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: .save {r4, r5, r7, lr}
-; CHECK-NEXT: push {r4, r5, r7, lr}
+; CHECK-NEXT: .save {r4, lr}
+; CHECK-NEXT: push {r4, lr}
; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: beq .LBB0_6
-; CHECK-NEXT: @ %bb.1: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r2, r3, lsl #1
-; CHECK-NEXT: add.w r4, r1, r3, lsl #1
-; CHECK-NEXT: cmp r5, r1
+; CHECK-NEXT: it eq
+; CHECK-NEXT: popeq {r4, pc}
+; CHECK-NEXT: .LBB0_1: @ %vector.memcheck
+; CHECK-NEXT: add.w r12, r1, r3, lsl #1
+; CHECK-NEXT: add.w lr, r2, r3, lsl #1
+; CHECK-NEXT: cmp r12, r2
+; CHECK-NEXT: add.w r4, r0, r3, lsl #1
; CHECK-NEXT: cset r12, hi
-; CHECK-NEXT: cmp r4, r2
+; CHECK-NEXT: cmp lr, r1
+; CHECK-NEXT: csel r12, zr, r12, ls
+; CHECK-NEXT: cmp lr, r0
; CHECK-NEXT: cset lr, hi
-; CHECK-NEXT: cmp r5, r0
-; CHECK-NEXT: add.w r5, r0, r3, lsl #1
+; CHECK-NEXT: cmp r4, r2
; CHECK-NEXT: cset r4, hi
-; CHECK-NEXT: cmp r5, r2
-; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: tst r5, r4
+; CHECK-NEXT: tst.w r4, lr
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r5, lr, r12
+; CHECK-NEXT: cmpeq.w r12, #0
; CHECK-NEXT: beq .LBB0_4
; CHECK-NEXT: @ %bb.2: @ %while.body.preheader
; CHECK-NEXT: dls lr, r3
@@ -47,7 +48,7 @@ define void @tailpred(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSr
; CHECK-NEXT: vstrh.16 q0, [r2], #16
; CHECK-NEXT: letp lr, .LBB0_5
; CHECK-NEXT: .LBB0_6: @ %while.end
-; CHECK-NEXT: pop {r4, r5, r7, pc}
+; CHECK-NEXT: pop {r4, pc}
entry:
%cmp.not6 = icmp eq i32 %blockSize, 0
br i1 %cmp.not6, label %while.end, label %vector.memcheck
@@ -114,20 +115,20 @@ define void @notailpred(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK-NEXT: cmp r3, #8
; CHECK-NEXT: blo .LBB1_3
; CHECK-NEXT: @ %bb.2: @ %vector.memcheck
-; CHECK-NEXT: add.w r5, r2, r3, lsl #1
-; CHECK-NEXT: add.w r6, r1, r3, lsl #1
-; CHECK-NEXT: cmp r5, r1
-; CHECK-NEXT: add.w r4, r0, r3, lsl #1
+; CHECK-NEXT: add.w r7, r1, r3, lsl #1
+; CHECK-NEXT: add.w r6, r2, r3, lsl #1
+; CHECK-NEXT: cmp r7, r2
+; CHECK-NEXT: add.w r5, r0, r3, lsl #1
; CHECK-NEXT: cset r7, hi
-; CHECK-NEXT: cmp r6, r2
+; CHECK-NEXT: cmp r6, r1
+; CHECK-NEXT: csel r7, zr, r7, ls
+; CHECK-NEXT: cmp r6, r0
; CHECK-NEXT: cset r6, hi
-; CHECK-NEXT: cmp r5, r0
+; CHECK-NEXT: cmp r5, r2
; CHECK-NEXT: cset r5, hi
-; CHECK-NEXT: cmp r4, r2
-; CHECK-NEXT: cset r4, hi
-; CHECK-NEXT: tst r4, r5
+; CHECK-NEXT: tst r5, r6
; CHECK-NEXT: it eq
-; CHECK-NEXT: andseq.w r7, r7, r6
+; CHECK-NEXT: cmpeq r7, #0
; CHECK-NEXT: beq .LBB1_7
; CHECK-NEXT: .LBB1_3:
; CHECK-NEXT: mov r5, r3
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
index e8d5eadabf7f9..42eec5e596a96 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
@@ -605,11 +605,11 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
; CHECK-NEXT: eors r0, r2
; CHECK-NEXT: orrs r0, r1
; CHECK-NEXT: vmov r1, r2, d0
-; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: orrs r1, r2
; CHECK-NEXT: vmov r12, r2, d5
; CHECK-NEXT: cset r1, eq
-; CHECK-NEXT: ands r0, r1
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: csel r0, zr, r1, ne
; CHECK-NEXT: movs r1, #0
; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #0, #8
@@ -618,10 +618,10 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
; CHECK-NEXT: eor.w r2, r3, r12
; CHECK-NEXT: orrs r0, r2
; CHECK-NEXT: vmov r2, r3, d1
-; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: orrs r2, r3
; CHECK-NEXT: cset r2, eq
-; CHECK-NEXT: ands r0, r2
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: csel r0, zr, r2, ne
; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #8, #8
; CHECK-NEXT: vmsr p0, r1
@@ -641,13 +641,13 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
; CHECK-NEXT: vmov r2, r3, d2
; CHECK-NEXT: eors r3, r1
; CHECK-NEXT: eors r2, r0
-; CHECK-NEXT: orrs r2, r3
+; CHECK-NEXT: orr.w r12, r2, r3
; CHECK-NEXT: vmov r3, r2, d0
-; CHECK-NEXT: cset r12, eq
; CHECK-NEXT: orrs r2, r3
; CHECK-NEXT: mov.w r3, #0
; CHECK-NEXT: cset r2, eq
-; CHECK-NEXT: and.w r2, r2, r12
+; CHECK-NEXT: cmp.w r12, #0
+; CHECK-NEXT: csel r2, zr, r2, ne
; CHECK-NEXT: rsbs r2, r2, #0
; CHECK-NEXT: bfi r3, r2, #0, #8
; CHECK-NEXT: vmov r12, r2, d3
@@ -655,10 +655,10 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
; CHECK-NEXT: eor.w r0, r0, r12
; CHECK-NEXT: orrs r0, r1
; CHECK-NEXT: vmov r1, r2, d1
-; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: orrs r1, r2
; CHECK-NEXT: cset r1, eq
-; CHECK-NEXT: ands r0, r1
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: csel r0, zr, r1, ne
; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r3, r0, #8, #8
; CHECK-NEXT: vmsr p0, r3
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
index 435ddf0a6e57b..b2e9877343bac 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -384,10 +384,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
; CHECK-NEXT: orrs r1, r2
; CHECK-NEXT: cset r1, eq
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r1, #1
-; CHECK-NEXT: rsbs r0, r1, #0
+; CHECK-NEXT: csinc r0, r1, zr, eq
; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #0, #8
; CHECK-NEXT: vmov r0, r2, d1
; CHECK-NEXT: orrs r0, r2
@@ -395,9 +394,8 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
; CHECK-NEXT: orrs r2, r3
; CHECK-NEXT: cset r2, eq
; CHECK-NEXT: cmp r0, #0
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r2, #1
-; CHECK-NEXT: rsbs r0, r2, #0
+; CHECK-NEXT: csinc r0, r2, zr, eq
+; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #8, #8
; CHECK-NEXT: vmsr p0, r1
; CHECK-NEXT: vpsel q0, q0, q1
@@ -421,12 +419,11 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
; CHECK-NEXT: vmov r1, r2, d0
; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: orrs r1, r2
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r0, #1
-; CHECK-NEXT: rsbs r0, r0, #0
+; CHECK-NEXT: vmov r12, r2, d5
+; CHECK-NEXT: csinc r0, r0, zr, eq
; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #0, #8
-; CHECK-NEXT: vmov r12, r2, d5
; CHECK-NEXT: vmov r3, r0, d3
; CHECK-NEXT: eors r0, r2
; CHECK-NEXT: eor.w r2, r3, r12
@@ -434,8 +431,7 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
; CHECK-NEXT: vmov r2, r3, d1
; CHECK-NEXT: cset r0, eq
; CHECK-NEXT: orrs r2, r3
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r0, #1
+; CHECK-NEXT: csinc r0, r0, zr, eq
; CHECK-NEXT: rsbs r0, r0, #0
; CHECK-NEXT: bfi r1, r0, #8, #8
; CHECK-NEXT: vmsr p0, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmp.ll b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
index f8e0a493b403e..020d06a3937a3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
@@ -455,38 +455,34 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
; CHECK-NEXT: vmov r1, r3, d1
; CHECK-NEXT: vmov.i32 q0, #0x0
; CHECK-NEXT: orrs r1, r3
-; CHECK-NEXT: vmov r3, s8
; CHECK-NEXT: csetm r1, eq
; CHECK-NEXT: bfi r2, r1, #8, #8
; CHECK-NEXT: vmsr p0, r2
+; CHECK-NEXT: vmov r2, s4
; CHECK-NEXT: vpsel q0, q0, q2
; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s8
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: vmov r2, s6
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #0, #8
; CHECK-NEXT: vmov r1, s2
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s10
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #8, #8
; CHECK-NEXT: vmsr p0, r0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
index 5802b0073f292..b30fa9b66274f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -47,22 +47,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
; CHECK-MVE-NEXT: cset r0, mi
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, s4
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, s7
; CHECK-MVE-NEXT: cset r1, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, s6
; CHECK-MVE-NEXT: cset r2, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, gt
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -238,22 +234,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
; CHECK-MVE-NEXT: cset r0, eq
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, s4
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, s7
; CHECK-MVE-NEXT: cset r1, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, s6
; CHECK-MVE-NEXT: cset r2, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, vs
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -617,14 +609,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s18, s12
; CHECK-MVE-NEXT: vmovx.f16 s4, s5
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s16, s18, s16
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s1
@@ -635,15 +625,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vcmp.f16 s1, s5
; CHECK-MVE-NEXT: vins.f16 s0, s16
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s2
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -655,14 +643,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: vmovx.f16 s6, s3
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s4
@@ -673,14 +659,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s6, s15
; CHECK-MVE-NEXT: vcmp.f16 s3, s7
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s4
@@ -1029,14 +1013,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s18, s12
; CHECK-MVE-NEXT: vmovx.f16 s4, s5
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s16, s18, s16
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s1
@@ -1047,15 +1029,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vcmp.f16 s1, s5
; CHECK-MVE-NEXT: vins.f16 s0, s16
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s2
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -1067,14 +1047,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: vmovx.f16 s6, s3
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s4
@@ -1085,14 +1063,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
; CHECK-MVE-NEXT: vmovx.f16 s6, s15
; CHECK-MVE-NEXT: vcmp.f16 s3, s7
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s4
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
index de6e85a8f5887..f35aa43f5a7aa 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -50,22 +50,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2
; CHECK-MVE-NEXT: cset r0, mi
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, s4
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, s4
; CHECK-MVE-NEXT: cset r1, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, s4
; CHECK-MVE-NEXT: cset r2, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, gt
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -256,22 +252,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2
; CHECK-MVE-NEXT: cset r0, eq
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, s4
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, s4
; CHECK-MVE-NEXT: cset r1, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, s4
; CHECK-MVE-NEXT: cset r2, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, vs
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -651,14 +643,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, s4
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s5, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s13
@@ -669,15 +659,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s1, s4
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s6
@@ -687,15 +675,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, s4
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s15
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s6
@@ -705,14 +691,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, s4
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s6
@@ -1043,14 +1027,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, s4
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s5, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s13
@@ -1061,15 +1043,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s1, s4
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s6
@@ -1079,15 +1059,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, s4
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s15
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s6
@@ -1097,14 +1075,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half %src2, <
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, s4
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s6
@@ -1687,22 +1663,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, float %sr
; CHECK-MVE-NEXT: cset r0, mi
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s0
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s3
; CHECK-MVE-NEXT: cset r1, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s2
; CHECK-MVE-NEXT: cset r2, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, gt
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -1893,22 +1865,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, float %sr
; CHECK-MVE-NEXT: cset r0, eq
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s0
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s3
; CHECK-MVE-NEXT: cset r1, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s4, s2
; CHECK-MVE-NEXT: cset r2, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, vs
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s15, s11
; CHECK-MVE-NEXT: cmp r3, #0
@@ -2288,14 +2256,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s5, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s13
@@ -2306,15 +2272,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s1
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s6
@@ -2324,15 +2288,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s2
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s15
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s6
@@ -2342,14 +2304,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s3
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s6
@@ -2680,14 +2640,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s5, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s12, s8
; CHECK-MVE-NEXT: vmovx.f16 s8, s13
@@ -2698,15 +2656,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s1
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s14
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s13, s9
; CHECK-MVE-NEXT: vins.f16 s1, s6
@@ -2716,15 +2672,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s2
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s15
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s14, s10
; CHECK-MVE-NEXT: vins.f16 s2, s6
@@ -2734,14 +2688,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, half %src2,
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s4, s3
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s6, s8, s6
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s15, s11
; CHECK-MVE-NEXT: vins.f16 s3, s6
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
index 809bf664fc95f..a48d45064f74b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -47,22 +47,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
; CHECK-MVE-NEXT: cset r0, mi
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, #0
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, #0
; CHECK-MVE-NEXT: cset r1, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, #0
; CHECK-MVE-NEXT: cset r2, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, gt
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s11, s7
; CHECK-MVE-NEXT: cmp r3, #0
@@ -238,22 +234,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
; CHECK-MVE-NEXT: cset r0, eq
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, #0
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, #0
; CHECK-MVE-NEXT: cset r1, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, #0
; CHECK-MVE-NEXT: cset r2, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, vs
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s11, s7
; CHECK-MVE-NEXT: cmp r3, #0
@@ -606,14 +598,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s12, s14, s12
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s8, s4
; CHECK-MVE-NEXT: vmovx.f16 s4, s1
@@ -624,15 +614,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vcmp.f16 s1, #0
; CHECK-MVE-NEXT: vins.f16 s0, s12
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s10
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s9, s5
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -642,14 +630,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s10, s6
; CHECK-MVE-NEXT: vmovx.f16 s6, s11
@@ -660,14 +646,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s11, s7
; CHECK-MVE-NEXT: vins.f16 s3, s4
@@ -983,14 +967,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s12, s14, s12
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s8, s4
; CHECK-MVE-NEXT: vmovx.f16 s4, s1
@@ -1001,15 +983,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vcmp.f16 s1, #0
; CHECK-MVE-NEXT: vins.f16 s0, s12
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s10
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s9, s5
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -1019,14 +999,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s10, s6
; CHECK-MVE-NEXT: vmovx.f16 s6, s11
@@ -1037,14 +1015,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s11, s7
; CHECK-MVE-NEXT: vins.f16 s3, s4
@@ -1600,22 +1576,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_one_v4f32(<4 x float> %src, <4 x floa
; CHECK-MVE-NEXT: cset r0, mi
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, #0
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, #0
; CHECK-MVE-NEXT: cset r1, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, #0
; CHECK-MVE-NEXT: cset r2, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, gt
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, gt
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s11, s7
; CHECK-MVE-NEXT: cmp r3, #0
@@ -1791,22 +1763,18 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_r_ueq_v4f32(<4 x float> %src, <4 x floa
; CHECK-MVE-NEXT: cset r0, eq
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s0, #0
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s3, #0
; CHECK-MVE-NEXT: cset r1, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r1, #1
+; CHECK-MVE-NEXT: csinc r1, r1, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f32 s2, #0
; CHECK-MVE-NEXT: cset r2, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r2, #1
+; CHECK-MVE-NEXT: csinc r2, r2, zr, vs
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r3, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r3, #1
+; CHECK-MVE-NEXT: csinc r3, r3, zr, vs
; CHECK-MVE-NEXT: cmp r2, #0
; CHECK-MVE-NEXT: vseleq.f32 s3, s11, s7
; CHECK-MVE-NEXT: cmp r3, #0
@@ -2159,14 +2127,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s12, s14, s12
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s8, s4
; CHECK-MVE-NEXT: vmovx.f16 s4, s1
@@ -2177,15 +2143,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vcmp.f16 s1, #0
; CHECK-MVE-NEXT: vins.f16 s0, s12
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s10
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s9, s5
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -2195,14 +2159,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s10, s6
; CHECK-MVE-NEXT: vmovx.f16 s6, s11
@@ -2213,14 +2175,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, #0
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, mi
-; CHECK-MVE-NEXT: it gt
-; CHECK-MVE-NEXT: movgt r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, gt
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s11, s7
; CHECK-MVE-NEXT: vins.f16 s3, s4
@@ -2536,14 +2496,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s0, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s12, s14, s12
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s0, s8, s4
; CHECK-MVE-NEXT: vmovx.f16 s4, s1
@@ -2554,15 +2512,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vcmp.f16 s1, #0
; CHECK-MVE-NEXT: vins.f16 s0, s12
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vmovx.f16 s8, s10
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s1, s9, s5
; CHECK-MVE-NEXT: vins.f16 s1, s4
@@ -2572,14 +2528,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s2, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s8, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s2, s10, s6
; CHECK-MVE-NEXT: vmovx.f16 s6, s11
@@ -2590,14 +2544,12 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half>
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: vcmp.f16 s3, #0
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s4, s6, s4
; CHECK-MVE-NEXT: vmrs APSR_nzcv, fpscr
; CHECK-MVE-NEXT: cset r0, eq
-; CHECK-MVE-NEXT: it vs
-; CHECK-MVE-NEXT: movvs r0, #1
+; CHECK-MVE-NEXT: csinc r0, r0, zr, vs
; CHECK-MVE-NEXT: cmp r0, #0
; CHECK-MVE-NEXT: vseleq.f16 s3, s11, s7
; CHECK-MVE-NEXT: vins.f16 s3, s4
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
index 707290f4f66cd..8312e91b51b69 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
@@ -496,38 +496,34 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
; CHECK-NEXT: vmov r1, r3, d1
; CHECK-NEXT: vmov.i32 q0, #0x0
; CHECK-NEXT: orrs r1, r3
-; CHECK-NEXT: vmov r3, s8
; CHECK-NEXT: csetm r1, eq
; CHECK-NEXT: bfi r2, r1, #8, #8
; CHECK-NEXT: vmsr p0, r2
+; CHECK-NEXT: vmov r2, s4
; CHECK-NEXT: vpsel q0, q0, q2
; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s8
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: vmov r2, s6
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #0, #8
; CHECK-NEXT: vmov r1, s2
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s10
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #8, #8
; CHECK-NEXT: vmsr p0, r0
@@ -1041,38 +1037,34 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_r_multi_v2i32(<2 x i64> %a, <2 x i32> %b,
; CHECK-NEXT: vmov r1, r3, d1
; CHECK-NEXT: vmov.i32 q0, #0x0
; CHECK-NEXT: orrs r1, r3
-; CHECK-NEXT: vmov r3, s8
; CHECK-NEXT: csetm r1, eq
; CHECK-NEXT: bfi r2, r1, #8, #8
; CHECK-NEXT: vmsr p0, r2
+; CHECK-NEXT: vmov r2, s4
; CHECK-NEXT: vpsel q0, q0, q2
; CHECK-NEXT: vmov r1, s0
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s4
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: vmov r3, s10
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s8
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: vmov r2, s6
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #0, #8
; CHECK-NEXT: vmov r1, s2
-; CHECK-NEXT: subs r2, r1, r3
-; CHECK-NEXT: asr.w r12, r1, #31
-; CHECK-NEXT: sbcs.w r2, r12, r3, asr #31
-; CHECK-NEXT: vmov r3, s6
-; CHECK-NEXT: cset r2, lt
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: cmp r1, #0
-; CHECK-NEXT: cset r1, ne
-; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: cset r3, ne
-; CHECK-NEXT: ands r1, r3
-; CHECK-NEXT: ands r1, r2
+; CHECK-NEXT: csel r12, zr, r2, eq
+; CHECK-NEXT: vmov r2, s10
+; CHECK-NEXT: asrs r3, r1, #31
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r3, r2, asr #31
+; CHECK-NEXT: csel r1, zr, r12, ge
; CHECK-NEXT: rsbs r1, r1, #0
; CHECK-NEXT: bfi r0, r1, #8, #8
; CHECK-NEXT: vmsr p0, r0
More information about the llvm-commits
mailing list