[llvm] e7dc083 - [ARM] Don't handle low overhead branches in AnalyzeBranch
David Green via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 18 09:16:19 PST 2021
Author: David Green
Date: 2021-01-18T17:16:07Z
New Revision: e7dc083a410f187e143138b4956993370626268b
URL: https://github.com/llvm/llvm-project/commit/e7dc083a410f187e143138b4956993370626268b
DIFF: https://github.com/llvm/llvm-project/commit/e7dc083a410f187e143138b4956993370626268b.diff
LOG: [ARM] Don't handle low overhead branches in AnalyzeBranch
It turns our that the BranchFolder and IfCvt does not like unanalyzable
branches that fall-through. This means that removing the unconditional
branches from the end of tail predicated instruction can run into
asserts and verifier issues.
This effectively reverts 372eb2bbb6fb903ce76266e659dfefbaee67722b, but
adds handling to t2DoLoopEndDec which are not branches, so can be safely
skipped.
Added:
llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
Modified:
llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll
llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
llvm/test/CodeGen/Thumb2/aligned-nonfallthrough.ll
llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll
llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll
llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 54586e0c256b..143bf6641e6f 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -367,15 +367,15 @@ bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
// Skip over DEBUG values, predicated nonterminators and speculation
// barrier terminators.
while (I->isDebugInstr() || !I->isTerminator() ||
- isSpeculationBarrierEndBBOpcode(I->getOpcode()) ){
+ isSpeculationBarrierEndBBOpcode(I->getOpcode()) ||
+ I->getOpcode() == ARM::t2DoLoopStartTP){
if (I == MBB.instr_begin())
return false;
--I;
}
if (isIndirectBranchOpcode(I->getOpcode()) ||
- isJumpTableBranchOpcode(I->getOpcode()) ||
- isLowOverheadTerminatorOpcode(I->getOpcode())) {
+ isJumpTableBranchOpcode(I->getOpcode())) {
// Indirect branches and jump tables can't be analyzed, but we still want
// to clean up any instructions at the tail of the basic block.
CantAnalyze = true;
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index ec574ad827a4..fec6ff7c2154 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -330,9 +330,9 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vdup.32 q1, r12
; CHECK-NEXT: vdup.32 q2, r12
; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT: b .LBB2_5
+; CHECK-NEXT: b .LBB2_4
; CHECK-NEXT: .LBB2_2: @ %cond.load25
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmovx.f16 s0, s28
; CHECK-NEXT: vmov r4, s28
; CHECK-NEXT: vmov r2, s0
@@ -344,7 +344,7 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov r2, s0
; CHECK-NEXT: vmov.16 q6[3], r2
; CHECK-NEXT: .LBB2_3: @ %else26
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmul.f16 q0, q6, q5
; CHECK-NEXT: adds r0, #8
; CHECK-NEXT: vcvtt.f32.f16 s23, s1
@@ -355,18 +355,9 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vcvtb.f32.f16 s20, s0
; CHECK-NEXT: vadd.f32 q5, q3, q5
; CHECK-NEXT: subs.w lr, lr, #1
-; CHECK-NEXT: bne .LBB2_5
-; CHECK-NEXT: @ %bb.4: @ %middle.block
-; CHECK-NEXT: vdup.32 q0, r12
-; CHECK-NEXT: vcmp.u32 cs, q0, q4
-; CHECK-NEXT: vpsel q0, q5, q3
-; CHECK-NEXT: vmov.f32 s4, s2
-; CHECK-NEXT: vmov.f32 s5, s3
-; CHECK-NEXT: vadd.f32 q0, q0, q1
-; CHECK-NEXT: vmov r0, s1
-; CHECK-NEXT: vadd.f32 q0, q0, r0
-; CHECK-NEXT: b .LBB2_23
-; CHECK-NEXT: .LBB2_5: @ %vector.body
+; CHECK-NEXT: bne .LBB2_4
+; CHECK-NEXT: b .LBB2_21
+; CHECK-NEXT: .LBB2_4: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload
; CHECK-NEXT: vmov q3, q5
@@ -388,13 +379,13 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: rsbs r4, r4, #0
; CHECK-NEXT: bfi r2, r4, #3, #1
; CHECK-NEXT: lsls r4, r2, #31
-; CHECK-NEXT: bne .LBB2_10
-; CHECK-NEXT: @ %bb.6: @ %else
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bne .LBB2_9
+; CHECK-NEXT: @ %bb.5: @ %else
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: lsls r4, r2, #30
-; CHECK-NEXT: bpl .LBB2_11
-; CHECK-NEXT: .LBB2_7: @ %cond.load6
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bpl .LBB2_10
+; CHECK-NEXT: .LBB2_6: @ %cond.load6
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vldr.16 s20, [r0, #2]
; CHECK-NEXT: vmov r5, s24
; CHECK-NEXT: vmovx.f16 s24, s25
@@ -406,25 +397,25 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov r4, s24
; CHECK-NEXT: vmov.16 q5[3], r4
; CHECK-NEXT: lsls r4, r2, #29
-; CHECK-NEXT: bmi .LBB2_12
-; CHECK-NEXT: .LBB2_8: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bmi .LBB2_11
+; CHECK-NEXT: .LBB2_7: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q6, q5
; CHECK-NEXT: lsls r2, r2, #28
-; CHECK-NEXT: bmi .LBB2_13
-; CHECK-NEXT: .LBB2_9: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bmi .LBB2_12
+; CHECK-NEXT: .LBB2_8: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q5, q6
-; CHECK-NEXT: b .LBB2_14
-; CHECK-NEXT: .LBB2_10: @ %cond.load
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: b .LBB2_13
+; CHECK-NEXT: .LBB2_9: @ %cond.load
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vldr.16 s24, [r0]
; CHECK-NEXT: lsls r4, r2, #30
-; CHECK-NEXT: bmi .LBB2_7
-; CHECK-NEXT: .LBB2_11: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bmi .LBB2_6
+; CHECK-NEXT: .LBB2_10: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q5, q6
; CHECK-NEXT: lsls r4, r2, #29
-; CHECK-NEXT: bpl .LBB2_8
-; CHECK-NEXT: .LBB2_12: @ %cond.load9
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bpl .LBB2_7
+; CHECK-NEXT: .LBB2_11: @ %cond.load9
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmovx.f16 s24, s20
; CHECK-NEXT: vmov r4, s20
; CHECK-NEXT: vldr.16 s28, [r0, #4]
@@ -437,9 +428,9 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov r4, s20
; CHECK-NEXT: vmov.16 q6[3], r4
; CHECK-NEXT: lsls r2, r2, #28
-; CHECK-NEXT: bpl .LBB2_9
-; CHECK-NEXT: .LBB2_13: @ %cond.load12
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bpl .LBB2_8
+; CHECK-NEXT: .LBB2_12: @ %cond.load12
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmovx.f16 s20, s24
; CHECK-NEXT: vmov r4, s24
; CHECK-NEXT: vmov r2, s20
@@ -450,8 +441,8 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov.16 q5[2], r2
; CHECK-NEXT: vmov r2, s24
; CHECK-NEXT: vmov.16 q5[3], r2
-; CHECK-NEXT: .LBB2_14: @ %else13
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: .LBB2_13: @ %else13
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vcmp.u32 cs, q2, q4
; CHECK-NEXT: @ implicit-def: $q7
; CHECK-NEXT: vmrs r4, p0
@@ -469,13 +460,13 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: rsbs r4, r4, #0
; CHECK-NEXT: bfi r2, r4, #3, #1
; CHECK-NEXT: lsls r4, r2, #31
-; CHECK-NEXT: bne .LBB2_18
-; CHECK-NEXT: @ %bb.15: @ %else17
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bne .LBB2_17
+; CHECK-NEXT: @ %bb.14: @ %else17
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: lsls r4, r2, #30
-; CHECK-NEXT: bpl .LBB2_19
-; CHECK-NEXT: .LBB2_16: @ %cond.load19
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bpl .LBB2_18
+; CHECK-NEXT: .LBB2_15: @ %cond.load19
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vldr.16 s24, [r1, #2]
; CHECK-NEXT: vmov r5, s28
; CHECK-NEXT: vmovx.f16 s28, s29
@@ -487,23 +478,23 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov r4, s28
; CHECK-NEXT: vmov.16 q6[3], r4
; CHECK-NEXT: lsls r4, r2, #29
-; CHECK-NEXT: bmi .LBB2_20
-; CHECK-NEXT: .LBB2_17: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bmi .LBB2_19
+; CHECK-NEXT: .LBB2_16: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q7, q6
; CHECK-NEXT: lsls r2, r2, #28
; CHECK-NEXT: bmi.w .LBB2_2
-; CHECK-NEXT: b .LBB2_21
-; CHECK-NEXT: .LBB2_18: @ %cond.load16
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: b .LBB2_20
+; CHECK-NEXT: .LBB2_17: @ %cond.load16
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vldr.16 s28, [r1]
; CHECK-NEXT: lsls r4, r2, #30
-; CHECK-NEXT: bmi .LBB2_16
-; CHECK-NEXT: .LBB2_19: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bmi .LBB2_15
+; CHECK-NEXT: .LBB2_18: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q6, q7
; CHECK-NEXT: lsls r4, r2, #29
-; CHECK-NEXT: bpl .LBB2_17
-; CHECK-NEXT: .LBB2_20: @ %cond.load22
-; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: bpl .LBB2_16
+; CHECK-NEXT: .LBB2_19: @ %cond.load22
+; CHECK-NEXT: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmovx.f16 s28, s24
; CHECK-NEXT: vmov r4, s24
; CHECK-NEXT: vldr.16 s0, [r1, #4]
@@ -517,9 +508,19 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
; CHECK-NEXT: vmov.16 q7[3], r4
; CHECK-NEXT: lsls r2, r2, #28
; CHECK-NEXT: bmi.w .LBB2_2
-; CHECK-NEXT: .LBB2_21: @ in Loop: Header=BB2_5 Depth=1
+; CHECK-NEXT: .LBB2_20: @ in Loop: Header=BB2_4 Depth=1
; CHECK-NEXT: vmov q6, q7
; CHECK-NEXT: b .LBB2_3
+; CHECK-NEXT: .LBB2_21: @ %middle.block
+; CHECK-NEXT: vdup.32 q0, r12
+; CHECK-NEXT: vcmp.u32 cs, q0, q4
+; CHECK-NEXT: vpsel q0, q5, q3
+; CHECK-NEXT: vmov.f32 s4, s2
+; CHECK-NEXT: vmov.f32 s5, s3
+; CHECK-NEXT: vadd.f32 q0, q0, q1
+; CHECK-NEXT: vmov r0, s1
+; CHECK-NEXT: vadd.f32 q0, q0, r0
+; CHECK-NEXT: b .LBB2_23
; CHECK-NEXT: .LBB2_22:
; CHECK-NEXT: vldr s0, .LCPI2_0
; CHECK-NEXT: .LBB2_23: @ %for.cond.cleanup
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index 872e9bd848ec..2b6c067b1f13 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -1468,7 +1468,7 @@ define arm_aapcs_vfpcc float @half_half_mac(half* nocapture readonly %a, half* n
; CHECK-NEXT: b .LBB9_6
; CHECK-NEXT: .LBB9_3:
; CHECK-NEXT: vldr s0, .LCPI9_0
-; CHECK-NEXT: pop {r4, r5, r7, pc}
+; CHECK-NEXT: b .LBB9_9
; CHECK-NEXT: .LBB9_4: @ %for.body.preheader.new
; CHECK-NEXT: bic r2, r2, #3
; CHECK-NEXT: movs r3, #1
@@ -1625,7 +1625,7 @@ define arm_aapcs_vfpcc float @half_half_acc(half* nocapture readonly %a, half* n
; CHECK-NEXT: b .LBB10_6
; CHECK-NEXT: .LBB10_3:
; CHECK-NEXT: vldr s0, .LCPI10_0
-; CHECK-NEXT: pop {r4, r5, r7, pc}
+; CHECK-NEXT: b .LBB10_9
; CHECK-NEXT: .LBB10_4: @ %for.body.preheader.new
; CHECK-NEXT: bic r2, r2, #3
; CHECK-NEXT: movs r3, #1
@@ -1782,7 +1782,7 @@ define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* n
; CHECK-NEXT: b .LBB11_6
; CHECK-NEXT: .LBB11_3:
; CHECK-NEXT: vldr s0, .LCPI11_0
-; CHECK-NEXT: pop {r4, r5, r6, pc}
+; CHECK-NEXT: b .LBB11_9
; CHECK-NEXT: .LBB11_4: @ %for.body.preheader.new
; CHECK-NEXT: bic r2, r2, #3
; CHECK-NEXT: movs r3, #1
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
index 9f0554cc4e23..4e72918f63f6 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/sibling-loops.ll
@@ -12,43 +12,41 @@ define arm_aapcs_vfpcc void @test(i16* noalias nocapture readonly %off, i16* noa
; CHECK-NEXT: lsl.w r12, r3, #1
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: mov r4, r1
-; CHECK-NEXT: b .LBB0_4
-; CHECK-NEXT: .LBB0_2: @ %for.body15.us
-; CHECK-NEXT: @ Parent Loop BB0_4 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: ldrh.w r7, [r0, r6, lsl #1]
-; CHECK-NEXT: ldrh.w r5, [r1, r6, lsl #1]
-; CHECK-NEXT: add r5, r7
-; CHECK-NEXT: strh.w r5, [r2, r6, lsl #1]
-; CHECK-NEXT: adds r6, #1
-; CHECK-NEXT: le lr, .LBB0_2
-; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup14.us
-; CHECK-NEXT: @ in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: adds r3, #1
-; CHECK-NEXT: add r2, r12
-; CHECK-NEXT: add r4, r12
-; CHECK-NEXT: cmp r3, r8
-; CHECK-NEXT: beq .LBB0_7
-; CHECK-NEXT: .LBB0_4: @ %for.cond1.preheader.us
+; CHECK-NEXT: .LBB0_2: @ %for.cond1.preheader.us
; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB0_3 Depth 2
; CHECK-NEXT: @ Child Loop BB0_5 Depth 2
-; CHECK-NEXT: @ Child Loop BB0_2 Depth 2
; CHECK-NEXT: dls lr, r8
; CHECK-NEXT: movs r6, #0
-; CHECK-NEXT: .LBB0_5: @ %for.body4.us
-; CHECK-NEXT: @ Parent Loop BB0_4 Depth=1
+; CHECK-NEXT: .LBB0_3: @ %for.body4.us
+; CHECK-NEXT: @ Parent Loop BB0_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: ldrh.w r5, [r0, r6, lsl #1]
; CHECK-NEXT: ldrh.w r7, [r1, r6, lsl #1]
; CHECK-NEXT: add r5, r7
; CHECK-NEXT: strh.w r5, [r4, r6, lsl #1]
; CHECK-NEXT: adds r6, #1
-; CHECK-NEXT: le lr, .LBB0_5
-; CHECK-NEXT: @ %bb.6: @ %for.body15.us.preheader
-; CHECK-NEXT: @ in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT: le lr, .LBB0_3
+; CHECK-NEXT: @ %bb.4: @ %for.body15.us.preheader
+; CHECK-NEXT: @ in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: dls lr, r8
; CHECK-NEXT: movs r6, #0
-; CHECK-NEXT: b .LBB0_2
+; CHECK-NEXT: .LBB0_5: @ %for.body15.us
+; CHECK-NEXT: @ Parent Loop BB0_2 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: ldrh.w r7, [r0, r6, lsl #1]
+; CHECK-NEXT: ldrh.w r5, [r1, r6, lsl #1]
+; CHECK-NEXT: add r5, r7
+; CHECK-NEXT: strh.w r5, [r2, r6, lsl #1]
+; CHECK-NEXT: adds r6, #1
+; CHECK-NEXT: le lr, .LBB0_5
+; CHECK-NEXT: @ %bb.6: @ %for.cond.cleanup14.us
+; CHECK-NEXT: @ in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT: adds r3, #1
+; CHECK-NEXT: add r2, r12
+; CHECK-NEXT: add r4, r12
+; CHECK-NEXT: cmp r3, r8
+; CHECK-NEXT: bne .LBB0_2
; CHECK-NEXT: .LBB0_7: @ %for.cond.cleanup
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, pc}
entry:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
index 0521594d1edc..1ea183d4a5ff 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
@@ -30,28 +30,11 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; ENABLED-NEXT: mov r9, r12
; ENABLED-NEXT: uxth r0, r0
; ENABLED-NEXT: rsbs r5, r0, #0
-; ENABLED-NEXT: b .LBB0_5
-; ENABLED-NEXT: .LBB0_2: @ %vector.body
-; ENABLED-NEXT: @ Parent Loop BB0_5 Depth=1
-; ENABLED-NEXT: @ => This Inner Loop Header: Depth=2
-; ENABLED-NEXT: vctp.32 r4
-; ENABLED-NEXT: vmov q0, q1
-; ENABLED-NEXT: vpstt
-; ENABLED-NEXT: vldrht.s32 q1, [r0], #8
-; ENABLED-NEXT: vldrht.s32 q2, [r7], #8
-; ENABLED-NEXT: mov lr, r6
-; ENABLED-NEXT: vmul.i32 q1, q2, q1
-; ENABLED-NEXT: subs r6, #1
-; ENABLED-NEXT: vshl.s32 q1, r5
-; ENABLED-NEXT: subs r4, #4
-; ENABLED-NEXT: vadd.i32 q1, q1, q0
-; ENABLED-NEXT: le lr, .LBB0_2
-; ENABLED-NEXT: @ %bb.3: @ %middle.block
-; ENABLED-NEXT: @ in Loop: Header=BB0_5 Depth=1
-; ENABLED-NEXT: vpsel q0, q1, q0
-; ENABLED-NEXT: vaddv.u32 r0, q0
-; ENABLED-NEXT: .LBB0_4: @ %for.end
-; ENABLED-NEXT: @ in Loop: Header=BB0_5 Depth=1
+; ENABLED-NEXT: b .LBB0_4
+; ENABLED-NEXT: .LBB0_2: @ in Loop: Header=BB0_4 Depth=1
+; ENABLED-NEXT: movs r0, #0
+; ENABLED-NEXT: .LBB0_3: @ %for.end
+; ENABLED-NEXT: @ in Loop: Header=BB0_4 Depth=1
; ENABLED-NEXT: lsrs r0, r0, #16
; ENABLED-NEXT: sub.w r9, r9, #1
; ENABLED-NEXT: strh.w r0, [r1, r8, lsl #1]
@@ -59,13 +42,13 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; ENABLED-NEXT: add.w r10, r10, #2
; ENABLED-NEXT: cmp r8, r3
; ENABLED-NEXT: beq .LBB0_8
-; ENABLED-NEXT: .LBB0_5: @ %for.body
+; ENABLED-NEXT: .LBB0_4: @ %for.body
; ENABLED-NEXT: @ =>This Loop Header: Depth=1
-; ENABLED-NEXT: @ Child Loop BB0_2 Depth 2
+; ENABLED-NEXT: @ Child Loop BB0_6 Depth 2
; ENABLED-NEXT: cmp r2, r8
-; ENABLED-NEXT: ble .LBB0_7
-; ENABLED-NEXT: @ %bb.6: @ %vector.ph
-; ENABLED-NEXT: @ in Loop: Header=BB0_5 Depth=1
+; ENABLED-NEXT: ble .LBB0_2
+; ENABLED-NEXT: @ %bb.5: @ %vector.ph
+; ENABLED-NEXT: @ in Loop: Header=BB0_4 Depth=1
; ENABLED-NEXT: bic r0, r9, #3
; ENABLED-NEXT: movs r7, #1
; ENABLED-NEXT: subs r0, #4
@@ -79,10 +62,26 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; ENABLED-NEXT: mov r7, r10
; ENABLED-NEXT: dls lr, r0
; ENABLED-NEXT: ldr r0, [sp] @ 4-byte Reload
-; ENABLED-NEXT: b .LBB0_2
-; ENABLED-NEXT: .LBB0_7: @ in Loop: Header=BB0_5 Depth=1
-; ENABLED-NEXT: movs r0, #0
-; ENABLED-NEXT: b .LBB0_4
+; ENABLED-NEXT: .LBB0_6: @ %vector.body
+; ENABLED-NEXT: @ Parent Loop BB0_4 Depth=1
+; ENABLED-NEXT: @ => This Inner Loop Header: Depth=2
+; ENABLED-NEXT: vctp.32 r4
+; ENABLED-NEXT: vmov q0, q1
+; ENABLED-NEXT: vpstt
+; ENABLED-NEXT: vldrht.s32 q1, [r0], #8
+; ENABLED-NEXT: vldrht.s32 q2, [r7], #8
+; ENABLED-NEXT: mov lr, r6
+; ENABLED-NEXT: vmul.i32 q1, q2, q1
+; ENABLED-NEXT: subs r6, #1
+; ENABLED-NEXT: vshl.s32 q1, r5
+; ENABLED-NEXT: subs r4, #4
+; ENABLED-NEXT: vadd.i32 q1, q1, q0
+; ENABLED-NEXT: le lr, .LBB0_6
+; ENABLED-NEXT: @ %bb.7: @ %middle.block
+; ENABLED-NEXT: @ in Loop: Header=BB0_4 Depth=1
+; ENABLED-NEXT: vpsel q0, q1, q0
+; ENABLED-NEXT: vaddv.u32 r0, q0
+; ENABLED-NEXT: b .LBB0_3
; ENABLED-NEXT: .LBB0_8: @ %for.end17
; ENABLED-NEXT: add sp, #4
; ENABLED-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
@@ -102,28 +101,11 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; NOREDUCTIONS-NEXT: mov r9, r12
; NOREDUCTIONS-NEXT: uxth r0, r0
; NOREDUCTIONS-NEXT: rsbs r5, r0, #0
-; NOREDUCTIONS-NEXT: b .LBB0_5
-; NOREDUCTIONS-NEXT: .LBB0_2: @ %vector.body
-; NOREDUCTIONS-NEXT: @ Parent Loop BB0_5 Depth=1
-; NOREDUCTIONS-NEXT: @ => This Inner Loop Header: Depth=2
-; NOREDUCTIONS-NEXT: vctp.32 r4
-; NOREDUCTIONS-NEXT: vmov q0, q1
-; NOREDUCTIONS-NEXT: vpstt
-; NOREDUCTIONS-NEXT: vldrht.s32 q1, [r0], #8
-; NOREDUCTIONS-NEXT: vldrht.s32 q2, [r7], #8
-; NOREDUCTIONS-NEXT: mov lr, r6
-; NOREDUCTIONS-NEXT: vmul.i32 q1, q2, q1
-; NOREDUCTIONS-NEXT: subs r6, #1
-; NOREDUCTIONS-NEXT: vshl.s32 q1, r5
-; NOREDUCTIONS-NEXT: subs r4, #4
-; NOREDUCTIONS-NEXT: vadd.i32 q1, q1, q0
-; NOREDUCTIONS-NEXT: le lr, .LBB0_2
-; NOREDUCTIONS-NEXT: @ %bb.3: @ %middle.block
-; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_5 Depth=1
-; NOREDUCTIONS-NEXT: vpsel q0, q1, q0
-; NOREDUCTIONS-NEXT: vaddv.u32 r0, q0
-; NOREDUCTIONS-NEXT: .LBB0_4: @ %for.end
-; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_5 Depth=1
+; NOREDUCTIONS-NEXT: b .LBB0_4
+; NOREDUCTIONS-NEXT: .LBB0_2: @ in Loop: Header=BB0_4 Depth=1
+; NOREDUCTIONS-NEXT: movs r0, #0
+; NOREDUCTIONS-NEXT: .LBB0_3: @ %for.end
+; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_4 Depth=1
; NOREDUCTIONS-NEXT: lsrs r0, r0, #16
; NOREDUCTIONS-NEXT: sub.w r9, r9, #1
; NOREDUCTIONS-NEXT: strh.w r0, [r1, r8, lsl #1]
@@ -131,13 +113,13 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; NOREDUCTIONS-NEXT: add.w r10, r10, #2
; NOREDUCTIONS-NEXT: cmp r8, r3
; NOREDUCTIONS-NEXT: beq .LBB0_8
-; NOREDUCTIONS-NEXT: .LBB0_5: @ %for.body
+; NOREDUCTIONS-NEXT: .LBB0_4: @ %for.body
; NOREDUCTIONS-NEXT: @ =>This Loop Header: Depth=1
-; NOREDUCTIONS-NEXT: @ Child Loop BB0_2 Depth 2
+; NOREDUCTIONS-NEXT: @ Child Loop BB0_6 Depth 2
; NOREDUCTIONS-NEXT: cmp r2, r8
-; NOREDUCTIONS-NEXT: ble .LBB0_7
-; NOREDUCTIONS-NEXT: @ %bb.6: @ %vector.ph
-; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_5 Depth=1
+; NOREDUCTIONS-NEXT: ble .LBB0_2
+; NOREDUCTIONS-NEXT: @ %bb.5: @ %vector.ph
+; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_4 Depth=1
; NOREDUCTIONS-NEXT: bic r0, r9, #3
; NOREDUCTIONS-NEXT: movs r7, #1
; NOREDUCTIONS-NEXT: subs r0, #4
@@ -151,10 +133,26 @@ define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input
; NOREDUCTIONS-NEXT: mov r7, r10
; NOREDUCTIONS-NEXT: dls lr, r0
; NOREDUCTIONS-NEXT: ldr r0, [sp] @ 4-byte Reload
-; NOREDUCTIONS-NEXT: b .LBB0_2
-; NOREDUCTIONS-NEXT: .LBB0_7: @ in Loop: Header=BB0_5 Depth=1
-; NOREDUCTIONS-NEXT: movs r0, #0
-; NOREDUCTIONS-NEXT: b .LBB0_4
+; NOREDUCTIONS-NEXT: .LBB0_6: @ %vector.body
+; NOREDUCTIONS-NEXT: @ Parent Loop BB0_4 Depth=1
+; NOREDUCTIONS-NEXT: @ => This Inner Loop Header: Depth=2
+; NOREDUCTIONS-NEXT: vctp.32 r4
+; NOREDUCTIONS-NEXT: vmov q0, q1
+; NOREDUCTIONS-NEXT: vpstt
+; NOREDUCTIONS-NEXT: vldrht.s32 q1, [r0], #8
+; NOREDUCTIONS-NEXT: vldrht.s32 q2, [r7], #8
+; NOREDUCTIONS-NEXT: mov lr, r6
+; NOREDUCTIONS-NEXT: vmul.i32 q1, q2, q1
+; NOREDUCTIONS-NEXT: subs r6, #1
+; NOREDUCTIONS-NEXT: vshl.s32 q1, r5
+; NOREDUCTIONS-NEXT: subs r4, #4
+; NOREDUCTIONS-NEXT: vadd.i32 q1, q1, q0
+; NOREDUCTIONS-NEXT: le lr, .LBB0_6
+; NOREDUCTIONS-NEXT: @ %bb.7: @ %middle.block
+; NOREDUCTIONS-NEXT: @ in Loop: Header=BB0_4 Depth=1
+; NOREDUCTIONS-NEXT: vpsel q0, q1, q0
+; NOREDUCTIONS-NEXT: vaddv.u32 r0, q0
+; NOREDUCTIONS-NEXT: b .LBB0_3
; NOREDUCTIONS-NEXT: .LBB0_8: @ %for.end17
; NOREDUCTIONS-NEXT: add sp, #4
; NOREDUCTIONS-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll
index 854f45bef455..c8001df58e8c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vcmp-vpst-combination.ll
@@ -66,7 +66,8 @@ define i32 @vcmp_new_vpst_combination(i32 %len, i32* nocapture readonly %arr) {
; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB1_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp7 = icmp sgt i32 %len, 0
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
index c40c30726a47..12e63f1b1edf 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
@@ -60,39 +60,39 @@ define void @nested(i32* nocapture readonly %x, i32* nocapture readnone %y, i32*
; CHECK-NEXT: ldr r5, [sp, #28]
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: movs r1, #0
-; CHECK-NEXT: b .LBB1_6
-; CHECK-NEXT: .LBB1_2: @ %do.body.preheader
-; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
+; CHECK-NEXT: b .LBB1_4
+; CHECK-NEXT: .LBB1_2: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: mov r4, r3
+; CHECK-NEXT: .LBB1_3: @ %if.end
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: str.w r4, [r2, r1, lsl #2]
+; CHECK-NEXT: adds r1, #1
+; CHECK-NEXT: cmp r1, r3
+; CHECK-NEXT: beq .LBB1_8
+; CHECK-NEXT: .LBB1_4: @ %for.body
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB1_6 Depth 2
+; CHECK-NEXT: adds r7, r5, #3
+; CHECK-NEXT: cmp.w r12, r7, lsr #2
+; CHECK-NEXT: beq .LBB1_2
+; CHECK-NEXT: @ %bb.5: @ %do.body.preheader
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
; CHECK-NEXT: bic r9, r7, #3
; CHECK-NEXT: mov r7, r5
; CHECK-NEXT: mov r4, r3
; CHECK-NEXT: add.w r8, r0, r9, lsl #2
; CHECK-NEXT: dlstp.32 lr, r5
-; CHECK-NEXT: .LBB1_3: @ %do.body
-; CHECK-NEXT: @ Parent Loop BB1_6 Depth=1
+; CHECK-NEXT: .LBB1_6: @ %do.body
+; CHECK-NEXT: @ Parent Loop BB1_4 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: vldrw.u32 q0, [r0], #16
; CHECK-NEXT: vaddva.s32 r4, q0
-; CHECK-NEXT: letp lr, .LBB1_3
-; CHECK-NEXT: @ %bb.4: @ %if.end.loopexit
-; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
+; CHECK-NEXT: letp lr, .LBB1_6
+; CHECK-NEXT: @ %bb.7: @ %if.end.loopexit
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
; CHECK-NEXT: sub.w r5, r5, r9
; CHECK-NEXT: mov r0, r8
-; CHECK-NEXT: .LBB1_5: @ %if.end
-; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: str.w r4, [r2, r1, lsl #2]
-; CHECK-NEXT: adds r1, #1
-; CHECK-NEXT: cmp r1, r3
-; CHECK-NEXT: beq .LBB1_8
-; CHECK-NEXT: .LBB1_6: @ %for.body
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB1_3 Depth 2
-; CHECK-NEXT: adds r7, r5, #3
-; CHECK-NEXT: cmp.w r12, r7, lsr #2
-; CHECK-NEXT: bne .LBB1_2
-; CHECK-NEXT: @ %bb.7: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: mov r4, r3
-; CHECK-NEXT: b .LBB1_5
+; CHECK-NEXT: b .LBB1_3
; CHECK-NEXT: .LBB1_8: @ %for.cond.cleanup
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, pc}
entry:
diff --git a/llvm/test/CodeGen/Thumb2/aligned-nonfallthrough.ll b/llvm/test/CodeGen/Thumb2/aligned-nonfallthrough.ll
index 043268d769d1..d1137145a0d3 100644
--- a/llvm/test/CodeGen/Thumb2/aligned-nonfallthrough.ll
+++ b/llvm/test/CodeGen/Thumb2/aligned-nonfallthrough.ll
@@ -15,6 +15,7 @@ define i32 @loop(i32* nocapture readonly %x) {
; CHECK-NEXT: ldr r2, [r0], #4
; CHECK-NEXT: add r1, r2
; CHECK-NEXT: le lr, .LBB0_1
+; CHECK-NEXT: .p2align 2
; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT: mov r0, r1
; CHECK-NEXT: pop {r7, pc}
@@ -54,6 +55,7 @@ define i64 @loopif(i32* nocapture readonly %x, i32 %y, i32 %n) {
; CHECK-NEXT: ldr r2, [r12], #4
; CHECK-NEXT: smlal r0, r3, r2, r1
; CHECK-NEXT: le lr, .LBB1_2
+; CHECK-NEXT: .p2align 2
; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: pop {r7, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
new file mode 100644
index 000000000000..fa1b81fdae4c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
@@ -0,0 +1,713 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+ at var_36 = hidden local_unnamed_addr global i8 0, align 1
+ at arr_61 = hidden local_unnamed_addr global [1 x i32] zeroinitializer, align 4
+
+define i32 @test(i8 zeroext %var_2, i16 signext %var_15, [18 x [22 x i8]]* %arr_60) {
+; CHECK-LABEL: test:
+; CHECK: @ %bb.0: @ %entry
+; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT: cmp r0, #0
+; CHECK-NEXT: beq.w .LBB0_10
+; CHECK-NEXT: @ %bb.1: @ %for.cond1.preheader
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: beq.w .LBB0_11
+; CHECK-NEXT: @ %bb.2: @ %for.cond1.preheader1
+; CHECK-NEXT: movw r8, :lower16:var_36
+; CHECK-NEXT: movw r0, #27476
+; CHECK-NEXT: addw r10, r2, #397
+; CHECK-NEXT: mov.w r9, #11
+; CHECK-NEXT: movt r8, :upper16:var_36
+; CHECK-NEXT: sdiv r1, r0, r1
+; CHECK-NEXT: mov.w r11, #0
+; CHECK-NEXT: .LBB0_3: @ %for.cond6.preheader
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB0_4 Depth 2
+; CHECK-NEXT: @ Child Loop BB0_6 Depth 2
+; CHECK-NEXT: @ Child Loop BB0_8 Depth 2
+; CHECK-NEXT: movs r0, #22
+; CHECK-NEXT: dls lr, r9
+; CHECK-NEXT: mla r7, r11, r0, r10
+; CHECK-NEXT: movw r0, :lower16:arr_61
+; CHECK-NEXT: movt r0, :upper16:arr_61
+; CHECK-NEXT: adds r0, #4
+; CHECK-NEXT: mov r3, r2
+; CHECK-NEXT: mov r6, r0
+; CHECK-NEXT: .LBB0_4: @ %for.body10
+; CHECK-NEXT: @ Parent Loop BB0_3 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: str r3, [r6, #-4]
+; CHECK-NEXT: add.w r12, r3, #396
+; CHECK-NEXT: ldrb r5, [r7, #-1]
+; CHECK-NEXT: add.w r3, r3, #792
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: str.w r12, [r6]
+; CHECK-NEXT: cset r5, ne
+; CHECK-NEXT: adds r6, #8
+; CHECK-NEXT: strb.w r5, [r8]
+; CHECK-NEXT: ldrb r5, [r7]
+; CHECK-NEXT: adds r7, #2
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: cset r5, ne
+; CHECK-NEXT: strb.w r5, [r8]
+; CHECK-NEXT: le lr, .LBB0_4
+; CHECK-NEXT: @ %bb.5: @ %for.cond.cleanup9
+; CHECK-NEXT: @ in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: add.w r3, r11, #1
+; CHECK-NEXT: movs r7, #22
+; CHECK-NEXT: dls lr, r9
+; CHECK-NEXT: mov r6, r0
+; CHECK-NEXT: uxtb r3, r3
+; CHECK-NEXT: smlabb r7, r3, r7, r10
+; CHECK-NEXT: mov r3, r2
+; CHECK-NEXT: .LBB0_6: @ %for.body10.1
+; CHECK-NEXT: @ Parent Loop BB0_3 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: str r3, [r6, #-4]
+; CHECK-NEXT: add.w r4, r3, #396
+; CHECK-NEXT: ldrb r5, [r7, #-1]
+; CHECK-NEXT: add.w r3, r3, #792
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: str r4, [r6]
+; CHECK-NEXT: cset r5, ne
+; CHECK-NEXT: adds r6, #8
+; CHECK-NEXT: strb.w r5, [r8]
+; CHECK-NEXT: ldrb r5, [r7]
+; CHECK-NEXT: adds r7, #2
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: cset r5, ne
+; CHECK-NEXT: strb.w r5, [r8]
+; CHECK-NEXT: le lr, .LBB0_6
+; CHECK-NEXT: @ %bb.7: @ %for.cond.cleanup9.1
+; CHECK-NEXT: @ in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: add.w r3, r11, #2
+; CHECK-NEXT: movs r7, #22
+; CHECK-NEXT: dls lr, r9
+; CHECK-NEXT: uxtb r3, r3
+; CHECK-NEXT: smlabb r7, r3, r7, r10
+; CHECK-NEXT: mov r3, r2
+; CHECK-NEXT: .LBB0_8: @ %for.body10.2
+; CHECK-NEXT: @ Parent Loop BB0_3 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: str r3, [r0, #-4]
+; CHECK-NEXT: ldrb r6, [r7, #-1]
+; CHECK-NEXT: cmp r6, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: add.w r6, r3, #396
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: str r6, [r0]
+; CHECK-NEXT: cset r6, ne
+; CHECK-NEXT: strb.w r6, [r8]
+; CHECK-NEXT: add.w r3, r3, #792
+; CHECK-NEXT: ldrb r6, [r7]
+; CHECK-NEXT: adds r0, #8
+; CHECK-NEXT: adds r7, #2
+; CHECK-NEXT: cmp r6, #0
+; CHECK-NEXT: ite ne
+; CHECK-NEXT: sxthne r5, r1
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: cmp r5, #0
+; CHECK-NEXT: cset r6, ne
+; CHECK-NEXT: strb.w r6, [r8]
+; CHECK-NEXT: le lr, .LBB0_8
+; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup9.2
+; CHECK-NEXT: @ in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: add.w r0, r11, #3
+; CHECK-NEXT: uxtb.w r11, r0
+; CHECK-NEXT: cmp.w r11, #18
+; CHECK-NEXT: it hs
+; CHECK-NEXT: movhs.w r11, #0
+; CHECK-NEXT: b .LBB0_3
+; CHECK-NEXT: .LBB0_10: @ %for.cond.cleanup
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT: .LBB0_11: @ %for.cond1.us.preheader
+; CHECK-NEXT: movw r0, :lower16:arr_61
+; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: movt r0, :upper16:arr_61
+; CHECK-NEXT: str r1, [r0, #84]
+; CHECK-NEXT: .inst.n 0xdefe
+entry:
+ %tobool.not = icmp eq i8 %var_2, 0
+ br i1 %tobool.not, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond1.preheader: ; preds = %entry
+ %cmp11.not = icmp eq [18 x [22 x i8]]* %arr_60, null
+ br i1 %cmp11.not, label %for.cond1.us.preheader, label %for.cond1
+
+for.cond1.us.preheader: ; preds = %for.cond1.preheader
+ store i32 0, i32* getelementptr ([1 x i32], [1 x i32]* @arr_61, i32 21, i32 0), align 4
+ call void @llvm.trap()
+ unreachable
+
+for.cond.cleanup: ; preds = %entry
+ ret i32 undef
+
+for.cond1: ; preds = %for.cond.cleanup9.2, %for.cond1.preheader
+ br label %for.cond6.preheader
+
+for.cond6.preheader: ; preds = %for.cond.cleanup9.2, %for.cond1
+ %conv45 = phi i32 [ 0, %for.cond1 ], [ %conv.2, %for.cond.cleanup9.2 ]
+ br label %for.body10
+
+for.cond.cleanup9: ; preds = %cond.end22.1
+ %add27 = add nuw nsw i32 %conv45, 1
+ %conv = and i32 %add27, 255
+ br label %for.body10.1
+
+for.body10: ; preds = %cond.end22.1, %for.cond6.preheader
+ %i_15.044 = phi i32 [ 0, %for.cond6.preheader ], [ %add.1, %cond.end22.1 ]
+ %arraydecay = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %i_15.044, i32 0
+ %0 = ptrtoint [22 x i8]* %arraydecay to i32
+ %arrayidx13 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %i_15.044
+ store i32 %0, i32* %arrayidx13, align 4
+ %arrayidx16 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv45, i32 %i_15.044
+ %1 = load i8, i8* %arrayidx16, align 1
+ %tobool18.not = icmp eq i8 %1, 0
+ br i1 %tobool18.not, label %cond.end22, label %cond.true19
+
+cond.true19: ; preds = %for.body10
+ %div43 = sdiv i16 27476, %var_15
+ %div.sext = sext i16 %div43 to i32
+ br label %cond.end22
+
+cond.end22: ; preds = %for.body10, %cond.true19
+ %cond23 = phi i32 [ %div.sext, %cond.true19 ], [ 0, %for.body10 ]
+ %tobool24 = icmp ne i32 %cond23, 0
+ %frombool = zext i1 %tobool24 to i8
+ store i8 %frombool, i8* @var_36, align 1
+ %add = or i32 %i_15.044, 1
+ %arraydecay.1 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %add, i32 0
+ %2 = ptrtoint [22 x i8]* %arraydecay.1 to i32
+ %arrayidx13.1 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %add
+ store i32 %2, i32* %arrayidx13.1, align 4
+ %arrayidx16.1 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv45, i32 %add
+ %3 = load i8, i8* %arrayidx16.1, align 1
+ %tobool18.not.1 = icmp eq i8 %3, 0
+ br i1 %tobool18.not.1, label %cond.end22.1, label %cond.true19.1
+
+cond.true19.1: ; preds = %cond.end22
+ %div43.1 = sdiv i16 27476, %var_15
+ %div.sext.1 = sext i16 %div43.1 to i32
+ br label %cond.end22.1
+
+cond.end22.1: ; preds = %cond.true19.1, %cond.end22
+ %cond23.1 = phi i32 [ %div.sext.1, %cond.true19.1 ], [ 0, %cond.end22 ]
+ %tobool24.1 = icmp ne i32 %cond23.1, 0
+ %frombool.1 = zext i1 %tobool24.1 to i8
+ store i8 %frombool.1, i8* @var_36, align 1
+ %add.1 = add nuw nsw i32 %i_15.044, 2
+ %exitcond105.not.1 = icmp eq i32 %add.1, 22
+ br i1 %exitcond105.not.1, label %for.cond.cleanup9, label %for.body10
+
+for.body10.1: ; preds = %cond.end22.1.1, %for.cond.cleanup9
+ %i_15.044.1 = phi i32 [ 0, %for.cond.cleanup9 ], [ %add.1.1, %cond.end22.1.1 ]
+ %arraydecay.1108 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %i_15.044.1, i32 0
+ %4 = ptrtoint [22 x i8]* %arraydecay.1108 to i32
+ %arrayidx13.1109 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %i_15.044.1
+ store i32 %4, i32* %arrayidx13.1109, align 4
+ %arrayidx16.1110 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv, i32 %i_15.044.1
+ %5 = load i8, i8* %arrayidx16.1110, align 1
+ %tobool18.not.1111 = icmp eq i8 %5, 0
+ br i1 %tobool18.not.1111, label %cond.end22.1119, label %cond.true19.1114
+
+cond.true19.1114: ; preds = %for.body10.1
+ %div43.1112 = sdiv i16 27476, %var_15
+ %div.sext.1113 = sext i16 %div43.1112 to i32
+ br label %cond.end22.1119
+
+cond.end22.1119: ; preds = %cond.true19.1114, %for.body10.1
+ %cond23.1115 = phi i32 [ %div.sext.1113, %cond.true19.1114 ], [ 0, %for.body10.1 ]
+ %tobool24.1116 = icmp ne i32 %cond23.1115, 0
+ %frombool.1117 = zext i1 %tobool24.1116 to i8
+ store i8 %frombool.1117, i8* @var_36, align 1
+ %add.1118 = or i32 %i_15.044.1, 1
+ %arraydecay.1.1 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %add.1118, i32 0
+ %6 = ptrtoint [22 x i8]* %arraydecay.1.1 to i32
+ %arrayidx13.1.1 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %add.1118
+ store i32 %6, i32* %arrayidx13.1.1, align 4
+ %arrayidx16.1.1 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv, i32 %add.1118
+ %7 = load i8, i8* %arrayidx16.1.1, align 1
+ %tobool18.not.1.1 = icmp eq i8 %7, 0
+ br i1 %tobool18.not.1.1, label %cond.end22.1.1, label %cond.true19.1.1
+
+cond.true19.1.1: ; preds = %cond.end22.1119
+ %div43.1.1 = sdiv i16 27476, %var_15
+ %div.sext.1.1 = sext i16 %div43.1.1 to i32
+ br label %cond.end22.1.1
+
+cond.end22.1.1: ; preds = %cond.true19.1.1, %cond.end22.1119
+ %cond23.1.1 = phi i32 [ %div.sext.1.1, %cond.true19.1.1 ], [ 0, %cond.end22.1119 ]
+ %tobool24.1.1 = icmp ne i32 %cond23.1.1, 0
+ %frombool.1.1 = zext i1 %tobool24.1.1 to i8
+ store i8 %frombool.1.1, i8* @var_36, align 1
+ %add.1.1 = add nuw nsw i32 %i_15.044.1, 2
+ %exitcond105.not.1.1 = icmp eq i32 %add.1.1, 22
+ br i1 %exitcond105.not.1.1, label %for.cond.cleanup9.1, label %for.body10.1
+
+for.cond.cleanup9.1: ; preds = %cond.end22.1.1
+ %add27.1 = add nuw nsw i32 %conv45, 2
+ %conv.1 = and i32 %add27.1, 255
+ br label %for.body10.2
+
+for.body10.2: ; preds = %cond.end22.1.2, %for.cond.cleanup9.1
+ %i_15.044.2 = phi i32 [ 0, %for.cond.cleanup9.1 ], [ %add.1.2, %cond.end22.1.2 ]
+ %arraydecay.2 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %i_15.044.2, i32 0
+ %8 = ptrtoint [22 x i8]* %arraydecay.2 to i32
+ %arrayidx13.2 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %i_15.044.2
+ store i32 %8, i32* %arrayidx13.2, align 4
+ %arrayidx16.2 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv.1, i32 %i_15.044.2
+ %9 = load i8, i8* %arrayidx16.2, align 1
+ %tobool18.not.2 = icmp eq i8 %9, 0
+ br i1 %tobool18.not.2, label %cond.end22.2, label %cond.true19.2
+
+cond.true19.2: ; preds = %for.body10.2
+ %div43.2 = sdiv i16 27476, %var_15
+ %div.sext.2 = sext i16 %div43.2 to i32
+ br label %cond.end22.2
+
+cond.end22.2: ; preds = %cond.true19.2, %for.body10.2
+ %cond23.2 = phi i32 [ %div.sext.2, %cond.true19.2 ], [ 0, %for.body10.2 ]
+ %tobool24.2 = icmp ne i32 %cond23.2, 0
+ %frombool.2 = zext i1 %tobool24.2 to i8
+ store i8 %frombool.2, i8* @var_36, align 1
+ %add.2 = or i32 %i_15.044.2, 1
+ %arraydecay.1.2 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 %add.2, i32 0
+ %10 = ptrtoint [22 x i8]* %arraydecay.1.2 to i32
+ %arrayidx13.1.2 = getelementptr inbounds [1 x i32], [1 x i32]* @arr_61, i32 0, i32 %add.2
+ store i32 %10, i32* %arrayidx13.1.2, align 4
+ %arrayidx16.1.2 = getelementptr inbounds [18 x [22 x i8]], [18 x [22 x i8]]* %arr_60, i32 1, i32 %conv.1, i32 %add.2
+ %11 = load i8, i8* %arrayidx16.1.2, align 1
+ %tobool18.not.1.2 = icmp eq i8 %11, 0
+ br i1 %tobool18.not.1.2, label %cond.end22.1.2, label %cond.true19.1.2
+
+cond.true19.1.2: ; preds = %cond.end22.2
+ %div43.1.2 = sdiv i16 27476, %var_15
+ %div.sext.1.2 = sext i16 %div43.1.2 to i32
+ br label %cond.end22.1.2
+
+cond.end22.1.2: ; preds = %cond.true19.1.2, %cond.end22.2
+ %cond23.1.2 = phi i32 [ %div.sext.1.2, %cond.true19.1.2 ], [ 0, %cond.end22.2 ]
+ %tobool24.1.2 = icmp ne i32 %cond23.1.2, 0
+ %frombool.1.2 = zext i1 %tobool24.1.2 to i8
+ store i8 %frombool.1.2, i8* @var_36, align 1
+ %add.1.2 = add nuw nsw i32 %i_15.044.2, 2
+ %exitcond105.not.1.2 = icmp eq i32 %add.1.2, 22
+ br i1 %exitcond105.not.1.2, label %for.cond.cleanup9.2, label %for.body10.2
+
+for.cond.cleanup9.2: ; preds = %cond.end22.1.2
+ %add27.2 = add nuw nsw i32 %conv45, 3
+ %conv.2 = and i32 %add27.2, 255
+ %cmp.2 = icmp ult i32 %conv.2, 18
+ br i1 %cmp.2, label %for.cond6.preheader, label %for.cond1
+}
+
+declare void @llvm.trap() #1
+
+
+ at b = hidden local_unnamed_addr global i32 0, align 4
+ at a = hidden local_unnamed_addr global i32 0, align 4
+ at c = hidden local_unnamed_addr global [1 x i32] zeroinitializer, align 4
+
+define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
+; CHECK-LABEL: d:
+; CHECK: @ %bb.0: @ %entry
+; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT: .pad #4
+; CHECK-NEXT: sub sp, #4
+; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: .pad #16
+; CHECK-NEXT: sub sp, #16
+; CHECK-NEXT: mov r12, r1
+; CHECK-NEXT: subs r1, r0, #1
+; CHECK-NEXT: sbcs r1, r12, #0
+; CHECK-NEXT: blt.w .LBB1_28
+; CHECK-NEXT: @ %bb.1: @ %for.cond2.preheader.lr.ph
+; CHECK-NEXT: movs r7, #1
+; CHECK-NEXT: cmp r2, #1
+; CHECK-NEXT: csel r3, r2, r7, lt
+; CHECK-NEXT: movw r6, #43691
+; CHECK-NEXT: mov r1, r3
+; CHECK-NEXT: cmp r3, #3
+; CHECK-NEXT: it ls
+; CHECK-NEXT: movls r1, #3
+; CHECK-NEXT: movt r6, #43690
+; CHECK-NEXT: subs r1, r1, r3
+; CHECK-NEXT: ldr r4, [sp, #112]
+; CHECK-NEXT: adds r1, #2
+; CHECK-NEXT: movw r10, :lower16:c
+; CHECK-NEXT: movt r10, :upper16:c
+; CHECK-NEXT: vmov.i32 q5, #0xc
+; CHECK-NEXT: umull r1, r5, r1, r6
+; CHECK-NEXT: vmov.i32 q6, #0xc
+; CHECK-NEXT: @ implicit-def: $r11
+; CHECK-NEXT: @ implicit-def: $r9
+; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT: str r0, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT: strd r2, r12, [sp] @ 8-byte Folded Spill
+; CHECK-NEXT: add.w r6, r7, r5, lsr #1
+; CHECK-NEXT: @ implicit-def: $r5
+; CHECK-NEXT: adr r1, .LCPI1_0
+; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: vadd.i32 q4, q0, r3
+; CHECK-NEXT: b .LBB1_4
+; CHECK-NEXT: .LBB1_2: @ %for.body6.preheader
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: mov r0, r9
+; CHECK-NEXT: cmn.w r9, #4
+; CHECK-NEXT: it le
+; CHECK-NEXT: mvnle r0, #3
+; CHECK-NEXT: movw r2, #18725
+; CHECK-NEXT: adds r0, #6
+; CHECK-NEXT: movt r2, #9362
+; CHECK-NEXT: sub.w r1, r0, r9
+; CHECK-NEXT: movs r5, #0
+; CHECK-NEXT: umull r2, r3, r1, r2
+; CHECK-NEXT: subs r2, r1, r3
+; CHECK-NEXT: add.w r2, r3, r2, lsr #1
+; CHECK-NEXT: lsrs r3, r2, #2
+; CHECK-NEXT: lsls r3, r3, #3
+; CHECK-NEXT: sub.w r2, r3, r2, lsr #2
+; CHECK-NEXT: subs r1, r2, r1
+; CHECK-NEXT: add r0, r1
+; CHECK-NEXT: add.w r9, r0, #7
+; CHECK-NEXT: ldrd r12, r0, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT: .LBB1_3: @ %for.cond.cleanup5
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: add.w r11, r11, #2
+; CHECK-NEXT: subs.w r1, r11, r0
+; CHECK-NEXT: asr.w r7, r11, #31
+; CHECK-NEXT: sbcs.w r1, r7, r12
+; CHECK-NEXT: bge.w .LBB1_28
+; CHECK-NEXT: .LBB1_4: @ %for.cond2.preheader
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB1_17 Depth 2
+; CHECK-NEXT: @ Child Loop BB1_8 Depth 2
+; CHECK-NEXT: @ Child Loop BB1_10 Depth 3
+; CHECK-NEXT: @ Child Loop BB1_12 Depth 3
+; CHECK-NEXT: cmp.w r9, #2
+; CHECK-NEXT: bgt .LBB1_3
+; CHECK-NEXT: @ %bb.5: @ %for.body6.lr.ph
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT: cmp r0, #5
+; CHECK-NEXT: bhi .LBB1_15
+; CHECK-NEXT: @ %bb.6: @ %for.body6.us.preheader
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: ldrd r2, r3, [sp, #104]
+; CHECK-NEXT: movs r0, #32
+; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: bl __aeabi_ldivmod
+; CHECK-NEXT: vdup.32 q0, r2
+; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT: ldrd r2, r12, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT: mov r7, r9
+; CHECK-NEXT: b .LBB1_8
+; CHECK-NEXT: .LBB1_7: @ %for.cond.cleanup17.us
+; CHECK-NEXT: @ in Loop: Header=BB1_8 Depth=2
+; CHECK-NEXT: add.w r9, r7, #7
+; CHECK-NEXT: cmn.w r7, #4
+; CHECK-NEXT: mov.w r5, #0
+; CHECK-NEXT: mov r7, r9
+; CHECK-NEXT: bge .LBB1_3
+; CHECK-NEXT: .LBB1_8: @ %for.body6.us
+; CHECK-NEXT: @ Parent Loop BB1_4 Depth=1
+; CHECK-NEXT: @ => This Loop Header: Depth=2
+; CHECK-NEXT: @ Child Loop BB1_10 Depth 3
+; CHECK-NEXT: @ Child Loop BB1_12 Depth 3
+; CHECK-NEXT: cbz r2, .LBB1_11
+; CHECK-NEXT: @ %bb.9: @ %for.body13.us51.preheader
+; CHECK-NEXT: @ in Loop: Header=BB1_8 Depth=2
+; CHECK-NEXT: movw r3, :lower16:a
+; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: movt r3, :upper16:a
+; CHECK-NEXT: vmov q1, q4
+; CHECK-NEXT: str r1, [r3]
+; CHECK-NEXT: movw r3, :lower16:b
+; CHECK-NEXT: movt r3, :upper16:b
+; CHECK-NEXT: str r1, [r3]
+; CHECK-NEXT: mov r1, r6
+; CHECK-NEXT: dlstp.32 lr, r6
+; CHECK-NEXT: .LBB1_10: @ %vector.body111
+; CHECK-NEXT: @ Parent Loop BB1_4 Depth=1
+; CHECK-NEXT: @ Parent Loop BB1_8 Depth=2
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
+; CHECK-NEXT: vshl.i32 q2, q1, #2
+; CHECK-NEXT: vadd.i32 q1, q1, q6
+; CHECK-NEXT: vadd.i32 q2, q2, r10
+; CHECK-NEXT: vstrw.32 q0, [q2]
+; CHECK-NEXT: letp lr, .LBB1_10
+; CHECK-NEXT: b .LBB1_13
+; CHECK-NEXT: .LBB1_11: @ %vector.body.preheader
+; CHECK-NEXT: @ in Loop: Header=BB1_8 Depth=2
+; CHECK-NEXT: vmov q1, q4
+; CHECK-NEXT: mov r1, r6
+; CHECK-NEXT: dlstp.32 lr, r6
+; CHECK-NEXT: .LBB1_12: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB1_4 Depth=1
+; CHECK-NEXT: @ Parent Loop BB1_8 Depth=2
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
+; CHECK-NEXT: vshl.i32 q2, q1, #2
+; CHECK-NEXT: vadd.i32 q1, q1, q5
+; CHECK-NEXT: vadd.i32 q2, q2, r10
+; CHECK-NEXT: vstrw.32 q0, [q2]
+; CHECK-NEXT: letp lr, .LBB1_12
+; CHECK-NEXT: .LBB1_13: @ %for.cond9.for.cond15.preheader_crit_edge.us
+; CHECK-NEXT: @ in Loop: Header=BB1_8 Depth=2
+; CHECK-NEXT: cmp r4, #0
+; CHECK-NEXT: beq .LBB1_7
+; CHECK-NEXT: @ %bb.14: @ %for.cond9.for.cond15.preheader_crit_edge.us
+; CHECK-NEXT: @ in Loop: Header=BB1_8 Depth=2
+; CHECK-NEXT: eor r1, r5, #1
+; CHECK-NEXT: lsls r1, r1, #31
+; CHECK-NEXT: bne .LBB1_7
+; CHECK-NEXT: b .LBB1_26
+; CHECK-NEXT: .LBB1_15: @ %for.body6.lr.ph.split
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: cmp r4, #0
+; CHECK-NEXT: beq.w .LBB1_2
+; CHECK-NEXT: @ %bb.16: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: ldrd r12, r0, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT: mov r7, r9
+; CHECK-NEXT: .LBB1_17: @ %for.body6.us60
+; CHECK-NEXT: @ Parent Loop BB1_4 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: lsls r1, r5, #31
+; CHECK-NEXT: bne .LBB1_27
+; CHECK-NEXT: @ %bb.18: @ %for.cond.cleanup17.us63
+; CHECK-NEXT: @ in Loop: Header=BB1_17 Depth=2
+; CHECK-NEXT: cmn.w r7, #4
+; CHECK-NEXT: bge .LBB1_22
+; CHECK-NEXT: @ %bb.19: @ %for.cond.cleanup17.us63.1
+; CHECK-NEXT: @ in Loop: Header=BB1_17 Depth=2
+; CHECK-NEXT: cmn.w r7, #12
+; CHECK-NEXT: bgt .LBB1_23
+; CHECK-NEXT: @ %bb.20: @ %for.cond.cleanup17.us63.2
+; CHECK-NEXT: @ in Loop: Header=BB1_17 Depth=2
+; CHECK-NEXT: cmn.w r7, #19
+; CHECK-NEXT: bgt .LBB1_24
+; CHECK-NEXT: @ %bb.21: @ %for.cond.cleanup17.us63.3
+; CHECK-NEXT: @ in Loop: Header=BB1_17 Depth=2
+; CHECK-NEXT: add.w r9, r7, #28
+; CHECK-NEXT: cmn.w r7, #25
+; CHECK-NEXT: mov.w r5, #0
+; CHECK-NEXT: mov r7, r9
+; CHECK-NEXT: blt .LBB1_17
+; CHECK-NEXT: b .LBB1_3
+; CHECK-NEXT: .LBB1_22: @ %for.cond.cleanup5.loopexit134.split.loop.exit139
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: add.w r9, r7, #7
+; CHECK-NEXT: b .LBB1_25
+; CHECK-NEXT: .LBB1_23: @ %for.cond.cleanup5.loopexit134.split.loop.exit137
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: add.w r9, r7, #14
+; CHECK-NEXT: b .LBB1_25
+; CHECK-NEXT: .LBB1_24: @ %for.cond.cleanup5.loopexit134.split.loop.exit135
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: add.w r9, r7, #21
+; CHECK-NEXT: .LBB1_25: @ %for.cond.cleanup5
+; CHECK-NEXT: @ in Loop: Header=BB1_4 Depth=1
+; CHECK-NEXT: movs r5, #0
+; CHECK-NEXT: b .LBB1_3
+; CHECK-NEXT: .LBB1_26: @ %for.inc19.us
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: b .LBB1_26
+; CHECK-NEXT: .LBB1_27: @ %for.inc19.us66
+; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: b .LBB1_27
+; CHECK-NEXT: .LBB1_28: @ %for.cond.cleanup
+; CHECK-NEXT: add sp, #16
+; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: add sp, #4
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: @ %bb.29:
+; CHECK-NEXT: .LCPI1_0:
+; CHECK-NEXT: .long 0 @ 0x0
+; CHECK-NEXT: .long 3 @ 0x3
+; CHECK-NEXT: .long 6 @ 0x6
+; CHECK-NEXT: .long 9 @ 0x9
+entry:
+ %cmp47 = icmp sgt i64 %e, 0
+ br i1 %cmp47, label %for.cond2.preheader.lr.ph, label %for.cond.cleanup
+
+for.cond2.preheader.lr.ph: ; preds = %entry
+ %cmp7.inv = icmp slt i32 %f, 1
+ %spec.select = select i1 %cmp7.inv, i32 %f, i32 1
+ %cmp1041 = icmp ult i32 %spec.select, 6
+ %tobool.not = icmp eq i32 %f, 0
+ %tobool20.not97 = icmp eq i32 %h, 0
+ %0 = icmp ugt i32 %spec.select, 3
+ %umax = select i1 %0, i32 %spec.select, i32 3
+ %1 = add i32 %umax, 2
+ %2 = sub i32 %1, %spec.select
+ %3 = udiv i32 %2, 3
+ %4 = add nuw nsw i32 %3, 1
+ %5 = icmp ugt i32 %spec.select, 3
+ %umax112 = select i1 %5, i32 %spec.select, i32 3
+ %6 = add i32 %umax112, 2
+ %7 = sub i32 %6, %spec.select
+ %8 = udiv i32 %7, 3
+ %9 = add nuw nsw i32 %8, 1
+ %n.rnd.up114 = add nuw nsw i32 %8, 4
+ %n.vec116 = and i32 %n.rnd.up114, 2147483644
+ %.splatinsert121 = insertelement <4 x i32> poison, i32 %spec.select, i32 0
+ %.splat122 = shufflevector <4 x i32> %.splatinsert121, <4 x i32> poison, <4 x i32> zeroinitializer
+ %induction123 = add <4 x i32> %.splat122, <i32 0, i32 3, i32 6, i32 9>
+ %n.rnd.up = add nuw nsw i32 %3, 4
+ %n.vec = and i32 %n.rnd.up, 2147483644
+ %.splatinsert = insertelement <4 x i32> poison, i32 %spec.select, i32 0
+ %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+ %induction = add <4 x i32> %.splat, <i32 0, i32 3, i32 6, i32 9>
+ br label %for.cond2.preheader
+
+for.cond2.preheader: ; preds = %for.cond2.preheader.lr.ph, %for.cond.cleanup5
+ %l.0.off050 = phi i1 [ undef, %for.cond2.preheader.lr.ph ], [ %l.1.off0.lcssa, %for.cond.cleanup5 ]
+ %i.049 = phi i32 [ undef, %for.cond2.preheader.lr.ph ], [ %add26, %for.cond.cleanup5 ]
+ %j.048 = phi i32 [ undef, %for.cond2.preheader.lr.ph ], [ %j.1.lcssa, %for.cond.cleanup5 ]
+ %cmp343 = icmp slt i32 %j.048, 3
+ br i1 %cmp343, label %for.body6.lr.ph, label %for.cond.cleanup5
+
+for.body6.lr.ph: ; preds = %for.cond2.preheader
+ br i1 %cmp1041, label %for.body6.us.preheader, label %for.body6.lr.ph.split
+
+for.body6.us.preheader: ; preds = %for.body6.lr.ph
+ %rem.us = srem i64 32, %g
+ %conv14.us = trunc i64 %rem.us to i32
+ %broadcast.splatinsert131 = insertelement <4 x i32> poison, i32 %conv14.us, i32 0
+ %broadcast.splat132 = shufflevector <4 x i32> %broadcast.splatinsert131, <4 x i32> poison, <4 x i32> zeroinitializer
+ %broadcast.splatinsert107 = insertelement <4 x i32> poison, i32 %conv14.us, i32 0
+ %broadcast.splat108 = shufflevector <4 x i32> %broadcast.splatinsert107, <4 x i32> poison, <4 x i32> zeroinitializer
+ br label %for.body6.us
+
+for.body6.us: ; preds = %for.body6.us.preheader, %for.cond.cleanup17.us
+ %l.1.off045.us = phi i1 [ false, %for.cond.cleanup17.us ], [ %l.0.off050, %for.body6.us.preheader ]
+ %j.144.us = phi i32 [ %add23.us, %for.cond.cleanup17.us ], [ %j.048, %for.body6.us.preheader ]
+ br i1 %tobool.not, label %vector.body, label %for.body13.us51.preheader
+
+vector.body: ; preds = %for.body6.us, %vector.body
+ %index = phi i32 [ %index.next, %vector.body ], [ 0, %for.body6.us ]
+ %vec.ind = phi <4 x i32> [ %vec.ind.next, %vector.body ], [ %induction, %for.body6.us ]
+ %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %4)
+ %10 = getelementptr inbounds [1 x i32], [1 x i32]* @c, i32 0, <4 x i32> %vec.ind
+ call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %broadcast.splat108, <4 x i32*> %10, i32 4, <4 x i1> %active.lane.mask)
+ %index.next = add i32 %index, 4
+ %vec.ind.next = add <4 x i32> %vec.ind, <i32 12, i32 12, i32 12, i32 12>
+ %11 = icmp eq i32 %index.next, %n.vec
+ br i1 %11, label %for.cond9.for.cond15.preheader_crit_edge.us, label %vector.body
+
+for.body13.us51.preheader: ; preds = %for.body6.us
+ store i32 0, i32* @b, align 4
+ store i32 0, i32* @a, align 4
+ br label %vector.body111
+
+vector.body111: ; preds = %vector.body111, %for.body13.us51.preheader
+ %index117 = phi i32 [ 0, %for.body13.us51.preheader ], [ %index.next118, %vector.body111 ]
+ %vec.ind124 = phi <4 x i32> [ %induction123, %for.body13.us51.preheader ], [ %vec.ind.next125, %vector.body111 ]
+ %active.lane.mask130 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index117, i32 %9)
+ %12 = getelementptr inbounds [1 x i32], [1 x i32]* @c, i32 0, <4 x i32> %vec.ind124
+ call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %broadcast.splat132, <4 x i32*> %12, i32 4, <4 x i1> %active.lane.mask130)
+ %index.next118 = add i32 %index117, 4
+ %vec.ind.next125 = add <4 x i32> %vec.ind124, <i32 12, i32 12, i32 12, i32 12>
+ %13 = icmp eq i32 %index.next118, %n.vec116
+ br i1 %13, label %for.cond9.for.cond15.preheader_crit_edge.us, label %vector.body111
+
+for.cond.cleanup17.us: ; preds = %for.cond9.for.cond15.preheader_crit_edge.us
+ %add23.us = add nsw i32 %j.144.us, 7
+ %cmp3.us = icmp slt i32 %j.144.us, -4
+ br i1 %cmp3.us, label %for.body6.us, label %for.cond.cleanup5
+
+for.inc19.us: ; preds = %for.cond9.for.cond15.preheader_crit_edge.us, %for.inc19.us
+ br label %for.inc19.us
+
+for.cond9.for.cond15.preheader_crit_edge.us: ; preds = %vector.body111, %vector.body
+ %l.1.off045.us.not = xor i1 %l.1.off045.us, true
+ %brmerge = or i1 %tobool20.not97, %l.1.off045.us.not
+ br i1 %brmerge, label %for.cond.cleanup17.us, label %for.inc19.us
+
+for.body6.lr.ph.split: ; preds = %for.body6.lr.ph
+ br i1 %tobool20.not97, label %for.body6.preheader, label %for.body6.us60
+
+for.body6.preheader: ; preds = %for.body6.lr.ph.split
+ %14 = icmp sgt i32 %j.048, -4
+ %smax = select i1 %14, i32 %j.048, i32 -4
+ %15 = add nsw i32 %smax, 6
+ %16 = sub i32 %15, %j.048
+ %17 = urem i32 %16, 7
+ %18 = sub i32 %16, %17
+ %19 = add nsw i32 %j.048, 7
+ %20 = add i32 %19, %18
+ br label %for.cond.cleanup5
+
+for.body6.us60: ; preds = %for.body6.lr.ph.split, %for.cond.cleanup17.us63.3
+ %l.1.off045.us61 = phi i1 [ false, %for.cond.cleanup17.us63.3 ], [ %l.0.off050, %for.body6.lr.ph.split ]
+ %j.144.us62 = phi i32 [ %add23.us64.3, %for.cond.cleanup17.us63.3 ], [ %j.048, %for.body6.lr.ph.split ]
+ br i1 %l.1.off045.us61, label %for.inc19.us66, label %for.cond.cleanup17.us63
+
+for.cond.cleanup17.us63: ; preds = %for.body6.us60
+ %cmp3.us65 = icmp slt i32 %j.144.us62, -4
+ br i1 %cmp3.us65, label %for.cond.cleanup17.us63.1, label %for.cond.cleanup5.loopexit134.split.loop.exit139
+
+for.inc19.us66: ; preds = %for.body6.us60, %for.inc19.us66
+ br label %for.inc19.us66
+
+for.cond.cleanup: ; preds = %for.cond.cleanup5, %entry
+ ret i32 undef
+
+for.cond.cleanup5.loopexit134.split.loop.exit135: ; preds = %for.cond.cleanup17.us63.2
+ %add23.us64.2.le = add nsw i32 %j.144.us62, 21
+ br label %for.cond.cleanup5
+
+for.cond.cleanup5.loopexit134.split.loop.exit137: ; preds = %for.cond.cleanup17.us63.1
+ %add23.us64.1.le = add nsw i32 %j.144.us62, 14
+ br label %for.cond.cleanup5
+
+for.cond.cleanup5.loopexit134.split.loop.exit139: ; preds = %for.cond.cleanup17.us63
+ %add23.us64.le = add nsw i32 %j.144.us62, 7
+ br label %for.cond.cleanup5
+
+for.cond.cleanup5: ; preds = %for.cond.cleanup5.loopexit134.split.loop.exit135, %for.cond.cleanup5.loopexit134.split.loop.exit137, %for.cond.cleanup5.loopexit134.split.loop.exit139, %for.cond.cleanup17.us63.3, %for.cond.cleanup17.us, %for.body6.preheader, %for.cond2.preheader
+ %j.1.lcssa = phi i32 [ %j.048, %for.cond2.preheader ], [ %20, %for.body6.preheader ], [ %add23.us, %for.cond.cleanup17.us ], [ %add23.us64.2.le, %for.cond.cleanup5.loopexit134.split.loop.exit135 ], [ %add23.us64.1.le, %for.cond.cleanup5.loopexit134.split.loop.exit137 ], [ %add23.us64.le, %for.cond.cleanup5.loopexit134.split.loop.exit139 ], [ %add23.us64.3, %for.cond.cleanup17.us63.3 ]
+ %l.1.off0.lcssa = phi i1 [ %l.0.off050, %for.cond2.preheader ], [ false, %for.body6.preheader ], [ false, %for.cond.cleanup17.us ], [ false, %for.cond.cleanup17.us63.3 ], [ false, %for.cond.cleanup5.loopexit134.split.loop.exit139 ], [ false, %for.cond.cleanup5.loopexit134.split.loop.exit137 ], [ false, %for.cond.cleanup5.loopexit134.split.loop.exit135 ]
+ %add26 = add nsw i32 %i.049, 2
+ %conv = sext i32 %add26 to i64
+ %cmp = icmp slt i64 %conv, %e
+ br i1 %cmp, label %for.cond2.preheader, label %for.cond.cleanup
+
+for.cond.cleanup17.us63.1: ; preds = %for.cond.cleanup17.us63
+ %cmp3.us65.1 = icmp slt i32 %j.144.us62, -11
+ br i1 %cmp3.us65.1, label %for.cond.cleanup17.us63.2, label %for.cond.cleanup5.loopexit134.split.loop.exit137
+
+for.cond.cleanup17.us63.2: ; preds = %for.cond.cleanup17.us63.1
+ %cmp3.us65.2 = icmp slt i32 %j.144.us62, -18
+ br i1 %cmp3.us65.2, label %for.cond.cleanup17.us63.3, label %for.cond.cleanup5.loopexit134.split.loop.exit135
+
+for.cond.cleanup17.us63.3: ; preds = %for.cond.cleanup17.us63.2
+ %add23.us64.3 = add nsw i32 %j.144.us62, 28
+ %cmp3.us65.3 = icmp slt i32 %j.144.us62, -25
+ br i1 %cmp3.us65.3, label %for.body6.us60, label %for.cond.cleanup5
+}
+
+declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #1
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32 immarg, <4 x i1>) #2
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index f0a1b2b001d1..5bf4ebf92f14 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -1102,20 +1102,9 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
; CHECK-NEXT: add.w r3, r12, #16
; CHECK-NEXT: str r4, [sp, #12] @ 4-byte Spill
; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT: b .LBB16_6
-; CHECK-NEXT: .LBB16_3: @ %while.body76
-; CHECK-NEXT: @ Parent Loop BB16_6 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: ldrh r1, [r6], #2
-; CHECK-NEXT: vldrh.u16 q1, [r0], #2
-; CHECK-NEXT: subs.w lr, lr, #1
-; CHECK-NEXT: vfma.f16 q0, q1, r1
-; CHECK-NEXT: bne .LBB16_3
-; CHECK-NEXT: @ %bb.4: @ %while.end.loopexit
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
-; CHECK-NEXT: add.w r5, r5, r8, lsl #1
-; CHECK-NEXT: .LBB16_5: @ %while.end
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: b .LBB16_4
+; CHECK-NEXT: .LBB16_3: @ %while.end
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload
; CHECK-NEXT: subs.w r9, r9, #1
; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload
@@ -1123,10 +1112,10 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
; CHECK-NEXT: add.w r0, r5, r0, lsl #1
; CHECK-NEXT: add.w r5, r0, #8
; CHECK-NEXT: beq.w .LBB16_12
-; CHECK-NEXT: .LBB16_6: @ %while.body
+; CHECK-NEXT: .LBB16_4: @ %while.body
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB16_8 Depth 2
-; CHECK-NEXT: @ Child Loop BB16_3 Depth 2
+; CHECK-NEXT: @ Child Loop BB16_6 Depth 2
+; CHECK-NEXT: @ Child Loop BB16_10 Depth 2
; CHECK-NEXT: vldrw.u32 q0, [r1], #8
; CHECK-NEXT: ldrh.w lr, [r12, #14]
; CHECK-NEXT: ldrh.w r0, [r12, #12]
@@ -1163,14 +1152,14 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
; CHECK-NEXT: adds r5, #16
; CHECK-NEXT: vfma.f16 q0, q1, lr
; CHECK-NEXT: cmp r0, #16
-; CHECK-NEXT: blo .LBB16_11
-; CHECK-NEXT: @ %bb.7: @ %for.body.preheader
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: blo .LBB16_7
+; CHECK-NEXT: @ %bb.5: @ %for.body.preheader
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: ldr r0, [sp] @ 4-byte Reload
; CHECK-NEXT: dls lr, r0
; CHECK-NEXT: ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: .LBB16_8: @ %for.body
-; CHECK-NEXT: @ Parent Loop BB16_6 Depth=1
+; CHECK-NEXT: .LBB16_6: @ %for.body
+; CHECK-NEXT: @ Parent Loop BB16_4 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: ldrh r0, [r6], #16
; CHECK-NEXT: vldrw.u32 q1, [r5]
@@ -1201,19 +1190,32 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, half* noca
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: adds r5, #16
; CHECK-NEXT: vfma.f16 q0, q1, r1
-; CHECK-NEXT: le lr, .LBB16_8
-; CHECK-NEXT: .LBB16_9: @ %for.end
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: le lr, .LBB16_6
+; CHECK-NEXT: b .LBB16_8
+; CHECK-NEXT: .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT: ldr r6, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: .LBB16_8: @ %for.end
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: cmp.w r8, #0
-; CHECK-NEXT: beq.w .LBB16_5
-; CHECK-NEXT: @ %bb.10: @ %while.body76.preheader
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: beq.w .LBB16_3
+; CHECK-NEXT: b .LBB16_9
+; CHECK-NEXT: .LBB16_9: @ %while.body76.preheader
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: mov r0, r5
; CHECK-NEXT: mov lr, r8
+; CHECK-NEXT: .LBB16_10: @ %while.body76
+; CHECK-NEXT: @ Parent Loop BB16_4 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: ldrh r1, [r6], #2
+; CHECK-NEXT: vldrh.u16 q1, [r0], #2
+; CHECK-NEXT: subs.w lr, lr, #1
+; CHECK-NEXT: vfma.f16 q0, q1, r1
+; CHECK-NEXT: bne .LBB16_10
+; CHECK-NEXT: b .LBB16_11
+; CHECK-NEXT: .LBB16_11: @ %while.end.loopexit
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT: add.w r5, r5, r8, lsl #1
; CHECK-NEXT: b .LBB16_3
-; CHECK-NEXT: .LBB16_11: @ in Loop: Header=BB16_6 Depth=1
-; CHECK-NEXT: ldr r6, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: b .LBB16_9
; CHECK-NEXT: .LBB16_12: @ %if.end
; CHECK-NEXT: add sp, #24
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 8344f6361114..3986b53cab21 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1074,30 +1074,19 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, float* noc
; CHECK-NEXT: str r4, [sp, #20] @ 4-byte Spill
; CHECK-NEXT: str r0, [sp, #8] @ 4-byte Spill
; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT: b .LBB16_6
-; CHECK-NEXT: .LBB16_3: @ %while.body76
-; CHECK-NEXT: @ Parent Loop BB16_6 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: ldr r0, [r7], #4
-; CHECK-NEXT: vldrw.u32 q1, [r6], #4
-; CHECK-NEXT: subs.w lr, lr, #1
-; CHECK-NEXT: vfma.f32 q0, q1, r0
-; CHECK-NEXT: bne .LBB16_3
-; CHECK-NEXT: @ %bb.4: @ %while.end.loopexit
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
-; CHECK-NEXT: add.w r5, r5, r3, lsl #2
-; CHECK-NEXT: .LBB16_5: @ %while.end
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: b .LBB16_4
+; CHECK-NEXT: .LBB16_3: @ %while.end
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: ldr r0, [sp, #16] @ 4-byte Reload
; CHECK-NEXT: subs.w r10, r10, #1
; CHECK-NEXT: vstrb.8 q0, [r2], #16
; CHECK-NEXT: add.w r0, r5, r0, lsl #2
; CHECK-NEXT: add.w r5, r0, #16
; CHECK-NEXT: beq .LBB16_12
-; CHECK-NEXT: .LBB16_6: @ %while.body
+; CHECK-NEXT: .LBB16_4: @ %while.body
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB16_8 Depth 2
-; CHECK-NEXT: @ Child Loop BB16_3 Depth 2
+; CHECK-NEXT: @ Child Loop BB16_6 Depth 2
+; CHECK-NEXT: @ Child Loop BB16_10 Depth 2
; CHECK-NEXT: vldrw.u32 q0, [r1], #16
; CHECK-NEXT: ldrd r7, r6, [r12]
; CHECK-NEXT: ldrd r0, r4, [r12, #8]
@@ -1123,14 +1112,14 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, float* noc
; CHECK-NEXT: vfma.f32 q0, q3, r11
; CHECK-NEXT: cmp r0, #16
; CHECK-NEXT: vfma.f32 q0, q1, r8
-; CHECK-NEXT: blo .LBB16_11
-; CHECK-NEXT: @ %bb.7: @ %for.body.preheader
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: blo .LBB16_7
+; CHECK-NEXT: @ %bb.5: @ %for.body.preheader
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload
; CHECK-NEXT: dls lr, r0
; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: .LBB16_8: @ %for.body
-; CHECK-NEXT: @ Parent Loop BB16_6 Depth=1
+; CHECK-NEXT: .LBB16_6: @ %for.body
+; CHECK-NEXT: @ Parent Loop BB16_4 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: ldm.w r7, {r0, r3, r4, r6}
; CHECK-NEXT: vldrw.u32 q1, [r5], #32
@@ -1153,21 +1142,34 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, float* noc
; CHECK-NEXT: adds r7, #32
; CHECK-NEXT: vfma.f32 q0, q3, r11
; CHECK-NEXT: vfma.f32 q0, q1, r9
-; CHECK-NEXT: le lr, .LBB16_8
-; CHECK-NEXT: .LBB16_9: @ %for.end
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: le lr, .LBB16_6
+; CHECK-NEXT: b .LBB16_8
+; CHECK-NEXT: .LBB16_7: @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT: .LBB16_8: @ %for.end
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: ldrd r9, r1, [sp, #24] @ 8-byte Folded Reload
; CHECK-NEXT: ldr r3, [sp, #12] @ 4-byte Reload
; CHECK-NEXT: cmp.w r3, #0
-; CHECK-NEXT: beq .LBB16_5
-; CHECK-NEXT: @ %bb.10: @ %while.body76.preheader
-; CHECK-NEXT: @ in Loop: Header=BB16_6 Depth=1
+; CHECK-NEXT: beq .LBB16_3
+; CHECK-NEXT: b .LBB16_9
+; CHECK-NEXT: .LBB16_9: @ %while.body76.preheader
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
; CHECK-NEXT: mov r6, r5
; CHECK-NEXT: mov lr, r3
+; CHECK-NEXT: .LBB16_10: @ %while.body76
+; CHECK-NEXT: @ Parent Loop BB16_4 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: ldr r0, [r7], #4
+; CHECK-NEXT: vldrw.u32 q1, [r6], #4
+; CHECK-NEXT: subs.w lr, lr, #1
+; CHECK-NEXT: vfma.f32 q0, q1, r0
+; CHECK-NEXT: bne .LBB16_10
+; CHECK-NEXT: b .LBB16_11
+; CHECK-NEXT: .LBB16_11: @ %while.end.loopexit
+; CHECK-NEXT: @ in Loop: Header=BB16_4 Depth=1
+; CHECK-NEXT: add.w r5, r5, r3, lsl #2
; CHECK-NEXT: b .LBB16_3
-; CHECK-NEXT: .LBB16_11: @ in Loop: Header=BB16_6 Depth=1
-; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: b .LBB16_9
; CHECK-NEXT: .LBB16_12: @ %if.end
; CHECK-NEXT: add sp, #32
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
@@ -1579,27 +1581,25 @@ define arm_aapcs_vfpcc void @fms(float* nocapture readonly %pSrc1, float* nocapt
; CHECK-NEXT: @ %bb.1: @ %do.body.preheader
; CHECK-NEXT: ldr.w r12, [sp, #20]
; CHECK-NEXT: lsr.w r5, lr, #2
-; CHECK-NEXT: b .LBB18_4
-; CHECK-NEXT: .LBB18_2: @ %while.body
-; CHECK-NEXT: @ Parent Loop BB18_4 Depth=1
+; CHECK-NEXT: .LBB18_2: @ %do.body
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB18_3 Depth 2
+; CHECK-NEXT: ldr r4, [r2]
+; CHECK-NEXT: dls lr, r5
+; CHECK-NEXT: vdup.32 q0, r4
+; CHECK-NEXT: .LBB18_3: @ %while.body
+; CHECK-NEXT: @ Parent Loop BB18_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: vldrw.u32 q2, [r0], #16
; CHECK-NEXT: vfms.f32 q2, q1, q0
; CHECK-NEXT: vstrb.8 q2, [r3], #16
-; CHECK-NEXT: le lr, .LBB18_2
-; CHECK-NEXT: @ %bb.3: @ %while.end
-; CHECK-NEXT: @ in Loop: Header=BB18_4 Depth=1
+; CHECK-NEXT: le lr, .LBB18_3
+; CHECK-NEXT: @ %bb.4: @ %while.end
+; CHECK-NEXT: @ in Loop: Header=BB18_2 Depth=1
; CHECK-NEXT: subs.w r12, r12, #1
; CHECK-NEXT: add.w r2, r2, #4
-; CHECK-NEXT: beq .LBB18_5
-; CHECK-NEXT: .LBB18_4: @ %do.body
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB18_2 Depth 2
-; CHECK-NEXT: ldr r4, [r2]
-; CHECK-NEXT: dls lr, r5
-; CHECK-NEXT: vdup.32 q0, r4
-; CHECK-NEXT: b .LBB18_2
+; CHECK-NEXT: bne .LBB18_2
; CHECK-NEXT: .LBB18_5: @ %do.end
; CHECK-NEXT: pop {r4, r5, r7, pc}
entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
index 81a6779f885d..c4f68959ecf4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
@@ -325,25 +325,23 @@ define arm_aapcs_vfpcc void @gather_inc_v4i32_simple(i32* noalias nocapture read
; CHECK-NEXT: adr r3, .LCPI8_0
; CHECK-NEXT: vldrw.u32 q0, [r3]
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: b .LBB8_4
-; CHECK-NEXT: .LBB8_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB8_4 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vldrw.u32 q2, [q1, #16]!
-; CHECK-NEXT: vstrb.8 q2, [r0], #16
-; CHECK-NEXT: le lr, .LBB8_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB8_4 Depth=1
-; CHECK-NEXT: cmp r12, r2
-; CHECK-NEXT: beq .LBB8_5
-; CHECK-NEXT: .LBB8_4: @ %vector.ph
+; CHECK-NEXT: .LBB8_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB8_2 Depth 2
+; CHECK-NEXT: @ Child Loop BB8_3 Depth 2
; CHECK-NEXT: dls lr, r4
; CHECK-NEXT: mov r0, r1
; CHECK-NEXT: vmov q1, q0
-; CHECK-NEXT: b .LBB8_2
-; CHECK-NEXT: .LBB8_5: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB8_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB8_2 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: vldrw.u32 q2, [q1, #16]!
+; CHECK-NEXT: vstrb.8 q2, [r0], #16
+; CHECK-NEXT: le lr, .LBB8_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB8_2 Depth=1
+; CHECK-NEXT: cmp r12, r2
+; CHECK-NEXT: bne .LBB8_2
+; CHECK-NEXT: @ %bb.5: @ %for.cond.cleanup
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.6:
@@ -404,9 +402,16 @@ define arm_aapcs_vfpcc void @gather_inc_v4i32_complex(i32* noalias nocapture rea
; CHECK-NEXT: vadd.i32 q1, q1, r0
; CHECK-NEXT: vadd.i32 q0, q0, r0
; CHECK-NEXT: vadd.i32 q2, q2, r0
-; CHECK-NEXT: b .LBB9_4
-; CHECK-NEXT: .LBB9_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB9_4 Depth=1
+; CHECK-NEXT: .LBB9_2: @ %vector.ph
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB9_3 Depth 2
+; CHECK-NEXT: dls lr, r3
+; CHECK-NEXT: mov r0, r1
+; CHECK-NEXT: vmov q3, q1
+; CHECK-NEXT: vmov q4, q0
+; CHECK-NEXT: vmov q5, q2
+; CHECK-NEXT: .LBB9_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB9_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: vldrw.u32 q6, [q5, #48]!
; CHECK-NEXT: vldrw.u32 q7, [q3, #48]!
@@ -414,20 +419,11 @@ define arm_aapcs_vfpcc void @gather_inc_v4i32_complex(i32* noalias nocapture rea
; CHECK-NEXT: vldrw.u32 q7, [q4, #48]!
; CHECK-NEXT: vadd.i32 q6, q6, q7
; CHECK-NEXT: vstrb.8 q6, [r0], #16
-; CHECK-NEXT: le lr, .LBB9_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB9_4 Depth=1
+; CHECK-NEXT: le lr, .LBB9_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB9_2 Depth=1
; CHECK-NEXT: cmp r12, r2
-; CHECK-NEXT: beq .LBB9_5
-; CHECK-NEXT: .LBB9_4: @ %vector.ph
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB9_2 Depth 2
-; CHECK-NEXT: dls lr, r3
-; CHECK-NEXT: mov r0, r1
-; CHECK-NEXT: vmov q3, q1
-; CHECK-NEXT: vmov q4, q0
-; CHECK-NEXT: vmov q5, q2
-; CHECK-NEXT: b .LBB9_2
+; CHECK-NEXT: bne .LBB9_2
; CHECK-NEXT: .LBB9_5: @ %for.cond.cleanup
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: pop {r4, r5, r7, pc}
@@ -502,25 +498,23 @@ define arm_aapcs_vfpcc void @gather_inc_v4i32_large(i32* noalias nocapture reado
; CHECK-NEXT: adr r3, .LCPI10_0
; CHECK-NEXT: vldrw.u32 q0, [r3]
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: b .LBB10_4
-; CHECK-NEXT: .LBB10_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB10_4 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vldrw.u32 q2, [q1, #508]!
-; CHECK-NEXT: vstrb.8 q2, [r0], #16
-; CHECK-NEXT: le lr, .LBB10_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB10_4 Depth=1
-; CHECK-NEXT: cmp r12, r2
-; CHECK-NEXT: beq .LBB10_5
-; CHECK-NEXT: .LBB10_4: @ %vector.ph
+; CHECK-NEXT: .LBB10_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB10_2 Depth 2
+; CHECK-NEXT: @ Child Loop BB10_3 Depth 2
; CHECK-NEXT: dls lr, r4
; CHECK-NEXT: mov r0, r1
; CHECK-NEXT: vmov q1, q0
-; CHECK-NEXT: b .LBB10_2
-; CHECK-NEXT: .LBB10_5: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB10_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB10_2 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: vldrw.u32 q2, [q1, #508]!
+; CHECK-NEXT: vstrb.8 q2, [r0], #16
+; CHECK-NEXT: le lr, .LBB10_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB10_2 Depth=1
+; CHECK-NEXT: cmp r12, r2
+; CHECK-NEXT: bne .LBB10_2
+; CHECK-NEXT: @ %bb.5: @ %for.cond.cleanup
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.6:
@@ -584,9 +578,15 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_simple(i16* noalias nocapture read
; CHECK-NEXT: adr r6, .LCPI11_0
; CHECK-NEXT: vldrw.u32 q0, [r6]
; CHECK-NEXT: str r1, [sp] @ 4-byte Spill
-; CHECK-NEXT: b .LBB11_4
-; CHECK-NEXT: .LBB11_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB11_4 Depth=1
+; CHECK-NEXT: .LBB11_2: @ %vector.ph
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB11_3 Depth 2
+; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT: vmov q2, q0
+; CHECK-NEXT: dls lr, r1
+; CHECK-NEXT: ldr r4, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: .LBB11_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: vmov.u16 r7, q2[6]
; CHECK-NEXT: vmov.u16 r3, q2[4]
@@ -632,19 +632,11 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_simple(i16* noalias nocapture read
; CHECK-NEXT: vmov.16 q3[6], r5
; CHECK-NEXT: vmov.16 q3[7], r6
; CHECK-NEXT: vstrb.8 q3, [r4], #16
-; CHECK-NEXT: le lr, .LBB11_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB11_4 Depth=1
+; CHECK-NEXT: le lr, .LBB11_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=1
; CHECK-NEXT: cmp r8, r2
-; CHECK-NEXT: beq .LBB11_5
-; CHECK-NEXT: .LBB11_4: @ %vector.ph
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB11_2 Depth 2
-; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT: vmov q2, q0
-; CHECK-NEXT: dls lr, r1
-; CHECK-NEXT: ldr r4, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: b .LBB11_2
+; CHECK-NEXT: bne .LBB11_2
; CHECK-NEXT: .LBB11_5: @ %for.cond.cleanup
; CHECK-NEXT: add sp, #8
; CHECK-NEXT: vpop {d8, d9}
@@ -725,9 +717,17 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_complex(i16* noalias nocapture rea
; CHECK-NEXT: str r1, [sp, #52] @ 4-byte Spill
; CHECK-NEXT: vstrw.32 q3, [sp, #64] @ 16-byte Spill
; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT: b .LBB12_4
-; CHECK-NEXT: .LBB12_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB12_4 Depth=1
+; CHECK-NEXT: .LBB12_2: @ %vector.ph
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB12_3 Depth 2
+; CHECK-NEXT: ldr r1, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT: dls lr, r1
+; CHECK-NEXT: ldr r4, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT: vldrw.u32 q7, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q5, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
+; CHECK-NEXT: .LBB12_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB12_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
; CHECK-NEXT: vmov.u16 r3, q5[2]
; CHECK-NEXT: vmov.u16 r5, q5[0]
@@ -864,22 +864,12 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_complex(i16* noalias nocapture rea
; CHECK-NEXT: vadd.i16 q0, q0, q2
; CHECK-NEXT: vadd.i16 q0, q0, q1
; CHECK-NEXT: vstrb.8 q0, [r4], #16
-; CHECK-NEXT: le lr, .LBB12_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB12_4 Depth=1
+; CHECK-NEXT: le lr, .LBB12_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT: ldr r1, [sp, #56] @ 4-byte Reload
; CHECK-NEXT: cmp r1, r2
-; CHECK-NEXT: beq .LBB12_5
-; CHECK-NEXT: .LBB12_4: @ %vector.ph
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB12_2 Depth 2
-; CHECK-NEXT: ldr r1, [sp, #52] @ 4-byte Reload
-; CHECK-NEXT: dls lr, r1
-; CHECK-NEXT: ldr r4, [sp, #60] @ 4-byte Reload
-; CHECK-NEXT: vldrw.u32 q7, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT: vldrw.u32 q5, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
-; CHECK-NEXT: b .LBB12_2
+; CHECK-NEXT: bne.w .LBB12_2
; CHECK-NEXT: .LBB12_5: @ %for.cond.cleanup
; CHECK-NEXT: add sp, #104
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
index 535affceaf3f..cfed9ccaebae 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
@@ -460,23 +460,29 @@ define dso_local void @arm_mat_mult_q31(i32* noalias nocapture readonly %A, i32*
; CHECK-NEXT: vldrw.u32 q2, [r7]
; CHECK-NEXT: vldrw.u32 q0, [r6]
; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
-; CHECK-NEXT: b .LBB9_2
-; CHECK-NEXT: .LBB9_1: @ %for.cond4.for.cond.cleanup6_crit_edge.us
-; CHECK-NEXT: @ in Loop: Header=BB9_2 Depth=1
-; CHECK-NEXT: add.w r8, r8, #1
-; CHECK-NEXT: cmp r8, r3
-; CHECK-NEXT: beq .LBB9_6
-; CHECK-NEXT: .LBB9_2: @ %for.cond8.preheader.us.us.preheader
+; CHECK-NEXT: .LBB9_1: @ %for.cond8.preheader.us.us.preheader
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB9_5 Depth 2
+; CHECK-NEXT: @ Child Loop BB9_2 Depth 2
; CHECK-NEXT: @ Child Loop BB9_3 Depth 3
; CHECK-NEXT: mul r11, r8, r9
; CHECK-NEXT: movs r5, #0
; CHECK-NEXT: mul r7, r8, r12
-; CHECK-NEXT: b .LBB9_5
+; CHECK-NEXT: .LBB9_2: @ %vector.ph
+; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1
+; CHECK-NEXT: @ => This Loop Header: Depth=2
+; CHECK-NEXT: @ Child Loop BB9_3 Depth 3
+; CHECK-NEXT: vdup.32 q5, r7
+; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT: vshl.i32 q5, q5, #2
+; CHECK-NEXT: vmov q6, q1
+; CHECK-NEXT: vadd.i32 q5, q5, r0
+; CHECK-NEXT: dls lr, r10
+; CHECK-NEXT: vmov.i32 q4, #0x0
+; CHECK-NEXT: vadd.i32 q5, q5, q0
+; CHECK-NEXT: vmlas.u32 q6, q2, r5
; CHECK-NEXT: .LBB9_3: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB9_2 Depth=1
-; CHECK-NEXT: @ Parent Loop BB9_5 Depth=2
+; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1
+; CHECK-NEXT: @ Parent Loop BB9_2 Depth=2
; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
; CHECK-NEXT: vadd.i32 q7, q6, q3
; CHECK-NEXT: vldrw.u32 q0, [r1, q6, uxtw #2]
@@ -486,28 +492,19 @@ define dso_local void @arm_mat_mult_q31(i32* noalias nocapture readonly %A, i32*
; CHECK-NEXT: vadd.i32 q4, q0, q4
; CHECK-NEXT: le lr, .LBB9_3
; CHECK-NEXT: @ %bb.4: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB9_5 Depth=2
+; CHECK-NEXT: @ in Loop: Header=BB9_2 Depth=2
; CHECK-NEXT: add.w r4, r5, r11
; CHECK-NEXT: adds r5, #1
; CHECK-NEXT: vaddv.u32 r6, q4
; CHECK-NEXT: cmp r5, r9
; CHECK-NEXT: str.w r6, [r2, r4, lsl #2]
-; CHECK-NEXT: beq .LBB9_1
-; CHECK-NEXT: .LBB9_5: @ %vector.ph
-; CHECK-NEXT: @ Parent Loop BB9_2 Depth=1
-; CHECK-NEXT: @ => This Loop Header: Depth=2
-; CHECK-NEXT: @ Child Loop BB9_3 Depth 3
-; CHECK-NEXT: vdup.32 q5, r7
-; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload
-; CHECK-NEXT: vshl.i32 q5, q5, #2
-; CHECK-NEXT: vmov q6, q1
-; CHECK-NEXT: vadd.i32 q5, q5, r0
-; CHECK-NEXT: dls lr, r10
-; CHECK-NEXT: vmov.i32 q4, #0x0
-; CHECK-NEXT: vadd.i32 q5, q5, q0
-; CHECK-NEXT: vmlas.u32 q6, q2, r5
-; CHECK-NEXT: b .LBB9_3
-; CHECK-NEXT: .LBB9_6: @ %for.end25
+; CHECK-NEXT: bne .LBB9_2
+; CHECK-NEXT: @ %bb.5: @ %for.cond4.for.cond.cleanup6_crit_edge.us
+; CHECK-NEXT: @ in Loop: Header=BB9_1 Depth=1
+; CHECK-NEXT: add.w r8, r8, #1
+; CHECK-NEXT: cmp r8, r3
+; CHECK-NEXT: bne .LBB9_1
+; CHECK-NEXT: @ %bb.6: @ %for.end25
; CHECK-NEXT: add sp, #24
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
@@ -864,43 +861,36 @@ define hidden arm_aapcs_vfpcc i32 @arm_depthwise_conv_s8(i8* nocapture readonly
; CHECK-NEXT: movs r6, #11
; CHECK-NEXT: vshl.i32 q1, q1, #2
; CHECK-NEXT: movs r5, #0
-; CHECK-NEXT: b .LBB11_2
-; CHECK-NEXT: .LBB11_1: @ %for.cond.cleanup20.i
-; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=1
-; CHECK-NEXT: ldr r5, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: ldr r7, [sp, #148]
-; CHECK-NEXT: adds r5, #1
-; CHECK-NEXT: cmp r5, r7
-; CHECK-NEXT: it eq
-; CHECK-NEXT: moveq r5, #0
-; CHECK-NEXT: .LBB11_2: @ %for.body10.i
+; CHECK-NEXT: .LBB11_1: @ %for.body10.i
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB11_4 Depth 2
-; CHECK-NEXT: @ Child Loop BB11_9 Depth 3
-; CHECK-NEXT: @ Child Loop BB11_5 Depth 4
-; CHECK-NEXT: @ Child Loop BB11_6 Depth 5
+; CHECK-NEXT: @ Child Loop BB11_2 Depth 2
+; CHECK-NEXT: @ Child Loop BB11_3 Depth 3
+; CHECK-NEXT: @ Child Loop BB11_4 Depth 4
+; CHECK-NEXT: @ Child Loop BB11_5 Depth 5
; CHECK-NEXT: movs r7, #0
; CHECK-NEXT: str r5, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT: b .LBB11_4
-; CHECK-NEXT: .LBB11_3: @ %for.cond.cleanup26.i
-; CHECK-NEXT: @ in Loop: Header=BB11_4 Depth=2
-; CHECK-NEXT: adds r7, #1
-; CHECK-NEXT: cmp r7, r3
-; CHECK-NEXT: beq .LBB11_1
-; CHECK-NEXT: .LBB11_4: @ %for.cond22.preheader.i
-; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1
+; CHECK-NEXT: .LBB11_2: @ %for.cond22.preheader.i
+; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1
; CHECK-NEXT: @ => This Loop Header: Depth=2
-; CHECK-NEXT: @ Child Loop BB11_9 Depth 3
-; CHECK-NEXT: @ Child Loop BB11_5 Depth 4
-; CHECK-NEXT: @ Child Loop BB11_6 Depth 5
+; CHECK-NEXT: @ Child Loop BB11_3 Depth 3
+; CHECK-NEXT: @ Child Loop BB11_4 Depth 4
+; CHECK-NEXT: @ Child Loop BB11_5 Depth 5
; CHECK-NEXT: movs r5, #0
-; CHECK-NEXT: b .LBB11_9
-; CHECK-NEXT: .LBB11_5: @ %for.body78.us.i
-; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1
-; CHECK-NEXT: @ Parent Loop BB11_4 Depth=2
-; CHECK-NEXT: @ Parent Loop BB11_9 Depth=3
+; CHECK-NEXT: .LBB11_3: @ %for.body27.i
+; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1
+; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2
+; CHECK-NEXT: @ => This Loop Header: Depth=3
+; CHECK-NEXT: @ Child Loop BB11_4 Depth 4
+; CHECK-NEXT: @ Child Loop BB11_5 Depth 5
+; CHECK-NEXT: dls lr, r9
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov.w r11, #4
+; CHECK-NEXT: .LBB11_4: @ %for.body78.us.i
+; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1
+; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2
+; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3
; CHECK-NEXT: @ => This Loop Header: Depth=4
-; CHECK-NEXT: @ Child Loop BB11_6 Depth 5
+; CHECK-NEXT: @ Child Loop BB11_5 Depth 5
; CHECK-NEXT: mul r4, r11, r6
; CHECK-NEXT: vdup.32 q3, r5
; CHECK-NEXT: vdup.32 q2, r7
@@ -910,11 +900,11 @@ define hidden arm_aapcs_vfpcc i32 @arm_depthwise_conv_s8(i8* nocapture readonly
; CHECK-NEXT: vadd.i32 q4, q0, r4
; CHECK-NEXT: mov r4, r8
; CHECK-NEXT: vmla.u32 q2, q4, r2
-; CHECK-NEXT: .LBB11_6: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1
-; CHECK-NEXT: @ Parent Loop BB11_4 Depth=2
-; CHECK-NEXT: @ Parent Loop BB11_9 Depth=3
-; CHECK-NEXT: @ Parent Loop BB11_5 Depth=4
+; CHECK-NEXT: .LBB11_5: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1
+; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2
+; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3
+; CHECK-NEXT: @ Parent Loop BB11_4 Depth=4
; CHECK-NEXT: @ => This Inner Loop Header: Depth=5
; CHECK-NEXT: vldrb.s32 q6, [r0, q2]
; CHECK-NEXT: vadd.i32 q5, q2, q1
@@ -925,27 +915,31 @@ define hidden arm_aapcs_vfpcc i32 @arm_depthwise_conv_s8(i8* nocapture readonly
; CHECK-NEXT: vmov q3, q4
; CHECK-NEXT: vmlava.u32 r12, q2, q6
; CHECK-NEXT: vmov q2, q5
-; CHECK-NEXT: bne .LBB11_6
-; CHECK-NEXT: @ %bb.7: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB11_5 Depth=4
+; CHECK-NEXT: bne .LBB11_5
+; CHECK-NEXT: @ %bb.6: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB11_4 Depth=4
; CHECK-NEXT: add.w r11, r11, #1
-; CHECK-NEXT: le lr, .LBB11_5
-; CHECK-NEXT: @ %bb.8: @ %for.cond.cleanup77.i
-; CHECK-NEXT: @ in Loop: Header=BB11_9 Depth=3
+; CHECK-NEXT: le lr, .LBB11_4
+; CHECK-NEXT: @ %bb.7: @ %for.cond.cleanup77.i
+; CHECK-NEXT: @ in Loop: Header=BB11_3 Depth=3
; CHECK-NEXT: adds r5, #1
; CHECK-NEXT: add.w r10, r10, #1
; CHECK-NEXT: cmp r5, r2
-; CHECK-NEXT: beq .LBB11_3
-; CHECK-NEXT: .LBB11_9: @ %for.body27.i
-; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1
-; CHECK-NEXT: @ Parent Loop BB11_4 Depth=2
-; CHECK-NEXT: @ => This Loop Header: Depth=3
-; CHECK-NEXT: @ Child Loop BB11_5 Depth 4
-; CHECK-NEXT: @ Child Loop BB11_6 Depth 5
-; CHECK-NEXT: dls lr, r9
-; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: mov.w r11, #4
-; CHECK-NEXT: b .LBB11_5
+; CHECK-NEXT: bne .LBB11_3
+; CHECK-NEXT: @ %bb.8: @ %for.cond.cleanup26.i
+; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=2
+; CHECK-NEXT: adds r7, #1
+; CHECK-NEXT: cmp r7, r3
+; CHECK-NEXT: bne .LBB11_2
+; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup20.i
+; CHECK-NEXT: @ in Loop: Header=BB11_1 Depth=1
+; CHECK-NEXT: ldr r5, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: ldr r7, [sp, #148]
+; CHECK-NEXT: adds r5, #1
+; CHECK-NEXT: cmp r5, r7
+; CHECK-NEXT: it eq
+; CHECK-NEXT: moveq r5, #0
+; CHECK-NEXT: b .LBB11_1
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.10:
; CHECK-NEXT: .LCPI11_0:
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll b/llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll
index 74bdd64e976b..b7e1c340fc5e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-tailpred.ll
@@ -17,25 +17,23 @@ define arm_aapcs_vfpcc void @gather_inc_v4i32_simple(i32* noalias nocapture read
; CHECK-NEXT: adr r3, .LCPI0_0
; CHECK-NEXT: vldrw.u32 q0, [r3]
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: b .LBB0_4
-; CHECK-NEXT: .LBB0_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB0_4 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vldrw.u32 q2, [q1, #16]!
-; CHECK-NEXT: vstrb.8 q2, [r0], #16
-; CHECK-NEXT: le lr, .LBB0_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: cmp r12, r2
-; CHECK-NEXT: beq .LBB0_5
-; CHECK-NEXT: .LBB0_4: @ %vector.ph
+; CHECK-NEXT: .LBB0_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB0_2 Depth 2
+; CHECK-NEXT: @ Child Loop BB0_3 Depth 2
; CHECK-NEXT: dls lr, r4
; CHECK-NEXT: mov r0, r1
; CHECK-NEXT: vmov q1, q0
-; CHECK-NEXT: b .LBB0_2
-; CHECK-NEXT: .LBB0_5: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB0_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB0_2 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: vldrw.u32 q2, [q1, #16]!
+; CHECK-NEXT: vstrb.8 q2, [r0], #16
+; CHECK-NEXT: le lr, .LBB0_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT: cmp r12, r2
+; CHECK-NEXT: bne .LBB0_2
+; CHECK-NEXT: @ %bb.5: @ %for.cond.cleanup
; CHECK-NEXT: pop {r4, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.6:
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index 7da0903e08ed..95a0c9458c8e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -1866,7 +1866,8 @@ define arm_aapcs_vfpcc void @usatmul_4_q15(i16* nocapture readonly %pSrcA, i16*
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r6, lr}
; CHECK-NEXT: push {r4, r5, r6, lr}
-; CHECK-NEXT: cbz r3, .LBB11_8
+; CHECK-NEXT: cmp r3, #0
+; CHECK-NEXT: beq .LBB11_8
; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
; CHECK-NEXT: cmp r3, #3
; CHECK-NEXT: bhi .LBB11_3
@@ -2131,7 +2132,8 @@ define arm_aapcs_vfpcc void @ssatmul_4_q7(i8* nocapture readonly %pSrcA, i8* noc
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r6, lr}
; CHECK-NEXT: push {r4, r5, r6, lr}
-; CHECK-NEXT: cbz r3, .LBB13_8
+; CHECK-NEXT: cmp r3, #0
+; CHECK-NEXT: beq .LBB13_8
; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
; CHECK-NEXT: cmp r3, #3
; CHECK-NEXT: bhi .LBB13_3
diff --git a/llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll b/llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll
index 280d218e9337..006413638205 100644
--- a/llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-scatter-increment.ll
@@ -160,26 +160,24 @@ define arm_aapcs_vfpcc void @scatter_inc_v4i32_complex(<4 x i32> %data1, <4 x i3
; CHECK-NEXT: vadd.i32 q4, q3, r0
; CHECK-NEXT: vldrw.u32 q3, [r12]
; CHECK-NEXT: vadd.i32 q3, q3, r0
-; CHECK-NEXT: b .LBB3_4
-; CHECK-NEXT: .LBB3_2: @ %vector.body
-; CHECK-NEXT: @ Parent Loop BB3_4 Depth=1
-; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vstrw.32 q0, [q5, #48]!
-; CHECK-NEXT: vstrw.32 q1, [q6, #48]!
-; CHECK-NEXT: vstrw.32 q2, [q7, #48]!
-; CHECK-NEXT: le lr, .LBB3_2
-; CHECK-NEXT: @ %bb.3: @ %middle.block
-; CHECK-NEXT: @ in Loop: Header=BB3_4 Depth=1
-; CHECK-NEXT: cmp r2, r1
-; CHECK-NEXT: beq .LBB3_5
-; CHECK-NEXT: .LBB3_4: @ %vector.ph
+; CHECK-NEXT: .LBB3_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB3_2 Depth 2
+; CHECK-NEXT: @ Child Loop BB3_3 Depth 2
; CHECK-NEXT: dls lr, r3
; CHECK-NEXT: vmov q6, q4
; CHECK-NEXT: vldrw.u32 q7, [sp] @ 16-byte Reload
; CHECK-NEXT: vmov q5, q3
-; CHECK-NEXT: b .LBB3_2
+; CHECK-NEXT: .LBB3_3: @ %vector.body
+; CHECK-NEXT: @ Parent Loop BB3_2 Depth=1
+; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
+; CHECK-NEXT: vstrw.32 q0, [q5, #48]!
+; CHECK-NEXT: vstrw.32 q1, [q6, #48]!
+; CHECK-NEXT: vstrw.32 q2, [q7, #48]!
+; CHECK-NEXT: le lr, .LBB3_3
+; CHECK-NEXT: @ %bb.4: @ %middle.block
+; CHECK-NEXT: @ in Loop: Header=BB3_2 Depth=1
+; CHECK-NEXT: cmp r2, r1
+; CHECK-NEXT: bne .LBB3_2
; CHECK-NEXT: .LBB3_5: @ %for.cond.cleanup
; CHECK-NEXT: add sp, #24
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
index 7186db6cda89..728328ac9cba 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
@@ -18,7 +18,7 @@ define i32 @add_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB0_7
; CHECK-NEXT: .LBB0_3:
; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB0_9
; CHECK-NEXT: .LBB0_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -45,7 +45,7 @@ define i32 @add_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: ldr r1, [r2], #4
; CHECK-NEXT: add r0, r1
; CHECK-NEXT: le lr, .LBB0_8
-; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB0_9: @ %for.cond.cleanup
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6 = icmp sgt i32 %n, 0
@@ -206,8 +206,8 @@ define i32 @and_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: b .LBB2_7
; CHECK-NEXT: .LBB2_3:
-; CHECK-NEXT: mov.w r0, #-1
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: mov.w r2, #-1
+; CHECK-NEXT: b .LBB2_9
; CHECK-NEXT: .LBB2_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -306,8 +306,8 @@ define i32 @or_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r2, #0
; CHECK-NEXT: b .LBB3_7
; CHECK-NEXT: .LBB3_3:
-; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: b .LBB3_9
; CHECK-NEXT: .LBB3_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -406,8 +406,8 @@ define i32 @xor_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r2, #0
; CHECK-NEXT: b .LBB4_7
; CHECK-NEXT: .LBB4_3:
-; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: b .LBB4_9
; CHECK-NEXT: .LBB4_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -507,8 +507,7 @@ define float @fadd_f32(float* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB5_7
; CHECK-NEXT: .LBB5_3:
; CHECK-NEXT: vldr s0, .LCPI5_0
-; CHECK-NEXT: vmov r0, s0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB5_9
; CHECK-NEXT: .LBB5_4: @ %vector.ph
; CHECK-NEXT: bic r2, r1, #3
; CHECK-NEXT: movs r3, #1
@@ -609,8 +608,7 @@ define float @fmul_f32(float* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB6_7
; CHECK-NEXT: .LBB6_3:
; CHECK-NEXT: vmov.f32 s0, #1.000000e+00
-; CHECK-NEXT: vmov r0, s0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB6_9
; CHECK-NEXT: .LBB6_4: @ %vector.ph
; CHECK-NEXT: bic r2, r1, #3
; CHECK-NEXT: movs r3, #1
@@ -706,8 +704,8 @@ define i32 @smin_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: b .LBB7_7
; CHECK-NEXT: .LBB7_3:
-; CHECK-NEXT: mvn r0, #-2147483648
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: mvn r2, #-2147483648
+; CHECK-NEXT: b .LBB7_9
; CHECK-NEXT: .LBB7_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -806,7 +804,7 @@ define i32 @smin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB8_7
; CHECK-NEXT: .LBB8_3:
; CHECK-NEXT: mvn r0, #-2147483648
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB8_9
; CHECK-NEXT: .LBB8_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -834,7 +832,7 @@ define i32 @smin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: cmp r0, r1
; CHECK-NEXT: csel r0, r0, r1, lt
; CHECK-NEXT: le lr, .LBB8_8
-; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB8_9: @ %for.cond.cleanup
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6 = icmp sgt i32 %n, 0
@@ -902,8 +900,8 @@ define i32 @smax_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: b .LBB9_7
; CHECK-NEXT: .LBB9_3:
-; CHECK-NEXT: mov.w r0, #-2147483648
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: mov.w r2, #-2147483648
+; CHECK-NEXT: b .LBB9_9
; CHECK-NEXT: .LBB9_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1002,7 +1000,7 @@ define i32 @smax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB10_7
; CHECK-NEXT: .LBB10_3:
; CHECK-NEXT: mov.w r0, #-2147483648
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB10_9
; CHECK-NEXT: .LBB10_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1030,7 +1028,7 @@ define i32 @smax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: cmp r0, r1
; CHECK-NEXT: csel r0, r0, r1, gt
; CHECK-NEXT: le lr, .LBB10_8
-; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB10_9: @ %for.cond.cleanup
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6 = icmp sgt i32 %n, 0
@@ -1098,8 +1096,8 @@ define i32 @umin_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r3, #0
; CHECK-NEXT: b .LBB11_7
; CHECK-NEXT: .LBB11_3:
-; CHECK-NEXT: mov.w r0, #-1
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: mov.w r2, #-1
+; CHECK-NEXT: b .LBB11_9
; CHECK-NEXT: .LBB11_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1198,7 +1196,7 @@ define i32 @umin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB12_7
; CHECK-NEXT: .LBB12_3:
; CHECK-NEXT: mov.w r0, #-1
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB12_9
; CHECK-NEXT: .LBB12_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1226,7 +1224,7 @@ define i32 @umin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: cmp r0, r1
; CHECK-NEXT: csel r0, r0, r1, hi
; CHECK-NEXT: le lr, .LBB12_8
-; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB12_9: @ %for.cond.cleanup
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6 = icmp sgt i32 %n, 0
@@ -1294,8 +1292,8 @@ define i32 @umax_i32(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: movs r2, #0
; CHECK-NEXT: b .LBB13_7
; CHECK-NEXT: .LBB13_3:
-; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: b .LBB13_9
; CHECK-NEXT: .LBB13_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1394,7 +1392,7 @@ define i32 @umax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB14_7
; CHECK-NEXT: .LBB14_3:
; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB14_9
; CHECK-NEXT: .LBB14_4: @ %vector.ph
; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: movs r2, #1
@@ -1422,7 +1420,7 @@ define i32 @umax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: cmp r0, r1
; CHECK-NEXT: csel r0, r0, r1, hi
; CHECK-NEXT: le lr, .LBB14_8
-; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup
+; CHECK-NEXT: .LBB14_9: @ %for.cond.cleanup
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6 = icmp sgt i32 %n, 0
@@ -1491,8 +1489,7 @@ define float @fmin_f32(float* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB15_7
; CHECK-NEXT: .LBB15_3:
; CHECK-NEXT: vldr s0, .LCPI15_0
-; CHECK-NEXT: vmov r0, s0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB15_9
; CHECK-NEXT: .LBB15_4: @ %vector.ph
; CHECK-NEXT: bic r2, r1, #3
; CHECK-NEXT: movs r3, #1
@@ -1597,8 +1594,7 @@ define float @fmax_f32(float* nocapture readonly %x, i32 %n) {
; CHECK-NEXT: b .LBB16_7
; CHECK-NEXT: .LBB16_3:
; CHECK-NEXT: vldr s0, .LCPI16_0
-; CHECK-NEXT: vmov r0, s0
-; CHECK-NEXT: pop {r7, pc}
+; CHECK-NEXT: b .LBB16_9
; CHECK-NEXT: .LBB16_4: @ %vector.ph
; CHECK-NEXT: bic r2, r1, #3
; CHECK-NEXT: movs r3, #1
@@ -1705,7 +1701,8 @@ define i32 @add4i32(i32* noalias nocapture readonly %x, i32 %n) {
; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB17_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6.not = icmp eq i32 %n, 0
@@ -1755,7 +1752,8 @@ define i32 @mla4i32(i32* noalias nocapture readonly %x, i32* noalias nocapture r
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB18_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp8.not = icmp eq i32 %n, 0
@@ -1808,7 +1806,8 @@ define i32 @add8i32(i16* noalias nocapture readonly %x, i32 %n) {
; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB19_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6.not = icmp eq i32 %n, 0
@@ -1859,7 +1858,8 @@ define i32 @mla8i32(i16* noalias nocapture readonly %x, i16* noalias nocapture r
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB20_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp9.not = icmp eq i32 %n, 0
@@ -1914,7 +1914,8 @@ define i32 @add16i32(i8* noalias nocapture readonly %x, i32 %n) {
; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB21_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp6.not = icmp eq i32 %n, 0
@@ -1965,7 +1966,8 @@ define i32 @mla16i32(i8* noalias nocapture readonly %x, i8* noalias nocapture re
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .LBB22_4:
-; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: pop {r7, pc}
entry:
%cmp9.not = icmp eq i32 %n, 0
@@ -2325,7 +2327,7 @@ define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: cbz r1, .LBB29_4
+; CHECK-NEXT: cbz r1, .LBB29_3
; CHECK-NEXT: @ %bb.1: @ %vector.ph
; CHECK-NEXT: movs r2, #0
; CHECK-NEXT: mov r3, r2
@@ -2335,14 +2337,14 @@ define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
; CHECK-NEXT: vldrw.u32 q0, [r0], #16
; CHECK-NEXT: vaddlva.s32 r2, r3, q0
; CHECK-NEXT: letp lr, .LBB29_2
-; CHECK-NEXT: .LBB29_3: @ %for.cond.cleanup
+; CHECK-NEXT: b .LBB29_4
+; CHECK-NEXT: .LBB29_3:
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: mov r3, r2
+; CHECK-NEXT: .LBB29_4: @ %for.cond.cleanup
; CHECK-NEXT: mov r0, r2
; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: pop {r7, pc}
-; CHECK-NEXT: .LBB29_4:
-; CHECK-NEXT: movs r2, #0
-; CHECK-NEXT: mov r3, r2
-; CHECK-NEXT: b .LBB29_3
entry:
%cmp6.not = icmp eq i32 %n, 0
br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
@@ -2378,7 +2380,7 @@ define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture r
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: cbz r2, .LBB30_4
+; CHECK-NEXT: cbz r2, .LBB30_3
; CHECK-NEXT: @ %bb.1: @ %vector.ph
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: mov r3, r12
@@ -2389,14 +2391,14 @@ define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture r
; CHECK-NEXT: vldrw.u32 q1, [r1], #16
; CHECK-NEXT: vmlalva.s32 r12, r3, q1, q0
; CHECK-NEXT: letp lr, .LBB30_2
-; CHECK-NEXT: .LBB30_3: @ %for.cond.cleanup
+; CHECK-NEXT: b .LBB30_4
+; CHECK-NEXT: .LBB30_3:
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov r3, r12
+; CHECK-NEXT: .LBB30_4: @ %for.cond.cleanup
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: pop {r7, pc}
-; CHECK-NEXT: .LBB30_4:
-; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: mov r3, r12
-; CHECK-NEXT: b .LBB30_3
entry:
%cmp9.not = icmp eq i32 %n, 0
br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
@@ -2437,7 +2439,7 @@ define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture r
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r7, lr}
; CHECK-NEXT: push {r7, lr}
-; CHECK-NEXT: cbz r2, .LBB31_4
+; CHECK-NEXT: cbz r2, .LBB31_3
; CHECK-NEXT: @ %bb.1: @ %vector.ph
; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: mov r3, r12
@@ -2448,14 +2450,14 @@ define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture r
; CHECK-NEXT: vldrh.u16 q1, [r1], #16
; CHECK-NEXT: vmlalva.s16 r12, r3, q1, q0
; CHECK-NEXT: letp lr, .LBB31_2
-; CHECK-NEXT: .LBB31_3: @ %for.cond.cleanup
+; CHECK-NEXT: b .LBB31_4
+; CHECK-NEXT: .LBB31_3:
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: mov r3, r12
+; CHECK-NEXT: .LBB31_4: @ %for.cond.cleanup
; CHECK-NEXT: mov r0, r12
; CHECK-NEXT: mov r1, r3
; CHECK-NEXT: pop {r7, pc}
-; CHECK-NEXT: .LBB31_4:
-; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: mov r3, r12
-; CHECK-NEXT: b .LBB31_3
entry:
%cmp9.not = icmp eq i32 %n, 0
br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
diff --git a/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll b/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
index 54ea792184b4..a3f1ea295f7c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
@@ -6,7 +6,8 @@ define void @arm_cmplx_mag_squared_f16(half* nocapture readonly %pSrc, half* noc
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r7, lr}
; CHECK-NEXT: push {r4, r5, r7, lr}
-; CHECK-NEXT: cbz r2, .LBB0_8
+; CHECK-NEXT: cmp r2, #0
+; CHECK-NEXT: beq .LBB0_8
; CHECK-NEXT: @ %bb.1: @ %while.body.preheader
; CHECK-NEXT: cmp r2, #8
; CHECK-NEXT: blo .LBB0_9
More information about the llvm-commits
mailing list