[llvm] [CodeGen] Split wide active lane mask into smaller masks if preferred (PR #202909)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 02:55:00 PDT 2026
https://github.com/kmclaughlin-arm created https://github.com/llvm/llvm-project/pull/202909
Adds optimizeGetActiveLaneMask to CodeGenPrepare, which will rewrite a
get.active.lane.mask used by a series of subvector extracts if this is the
form preferred by the target.
This PR is part of a series of changes leading to the use of wide active
lane masks as the canonical form in LoopVectorize when choosing to interleave
and tail-fold loops. https://github.com/llvm/llvm-project/pull/193757 is the original PR where this was discussed.
For all targets other than AArch64, the preference is to use multiple smaller
get.active.lane.mask intrinsics, which matches the output of LoopVectorize today
when interleaving unless wide active lane masks are forced. For AArch64,
prefer a wide mask + extracts when either SVE2p1 or SME2 is available.
The motivation for this PR is to leave the decision on which form of the IR
to use up to the target. This will remove the need for more complex decisions
and transformations during vectorisation to query whether it's best to create
a single mask + extracts or multiple smaller masks.
>From 8012eacf8d69473663c3842d3dcb0c5cea11bdc7 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 9 Jun 2026 09:17:00 +0000
Subject: [PATCH 1/2] Add get.active.lane.mask + extract tests
---
.../RISCV/rvv/get-active-lane-mask-extract.ll | 140 +++
.../Thumb2/get-active-lane-mask-extract.ll | 836 ++++++++++++++++++
2 files changed, 976 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
create mode 100644 llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
diff --git a/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
new file mode 100644
index 0000000000000..e6a514645ee28
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+
+define void @test_2x4bit_mask_with_extracts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x4bit_mask_with_extracts:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vsaddu.vx v8, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v8, a1
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vmv.v.i v8, 0
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT: srli a0, a0, 1
+; CHECK-NEXT: vslidedown.vx v8, v8, a0
+; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: tail use
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+ %v1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+ tail call void @use(<vscale x 4 x i1> %v0, <vscale x 4 x i1> %v1)
+ ret void
+}
+
+define void @test_2x8bit_mask_with_extracts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x8bit_mask_with_extracts:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: vsaddu.vx v16, v8, a0
+; CHECK-NEXT: vadd.vx v8, v8, a2
+; CHECK-NEXT: vmsltu.vx v0, v16, a1
+; CHECK-NEXT: vsaddu.vx v16, v8, a0
+; CHECK-NEXT: vmsltu.vx v8, v16, a1
+; CHECK-NEXT: tail use
+ %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+ %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+ tail call void @use(<vscale x 8 x i1> %v0, <vscale x 8 x i1> %v1)
+ ret void
+}
+
+define void @test_fixed_2x4bit_mask_with_extract(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_fixed_2x4bit_mask_with_extract:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vsaddu.vx v8, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v8, a1
+; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT: vmv.v.i v8, 0
+; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT: vsetivli zero, 4, e8, mf2, ta, ma
+; CHECK-NEXT: vslidedown.vi v8, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: tail use
+ %r = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 %i, i64 %n)
+ %v0 = call <4 x i1> @llvm.vector.extract.v4i1.v8i1.i64(<8 x i1> %r, i64 0)
+ %v1 = call <4 x i1> @llvm.vector.extract.v4i1.v8i1.i64(<8 x i1> %r, i64 4)
+ tail call void @use(<4 x i1> %v0, <4 x i1> %v1)
+ ret void
+}
+
+define void @test_4x2bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_4x2bit_mask_with_extracts_and_reinterpret_casts:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vsaddu.vx v8, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v8, a1
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bnez a0, .LBB3_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vmv.v.i v8, 0
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: vmerge.vim v10, v8, 1, v0
+; CHECK-NEXT: srli a1, a0, 2
+; CHECK-NEXT: srli a2, a0, 1
+; CHECK-NEXT: vslidedown.vx v8, v10, a1
+; CHECK-NEXT: vslidedown.vx v9, v10, a2
+; CHECK-NEXT: sub a0, a0, a1
+; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: vmsne.vi v9, v9, 0
+; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v10, a0
+; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT: vmsne.vi v10, v10, 0
+; CHECK-NEXT: tail use
+; CHECK-NEXT: .LBB3_2: # %if.end
+; CHECK-NEXT: ret
+entry:
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i64 %i, i64 %n)
+ %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+ %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 2)
+ %v2 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+ %v3 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 6)
+ %elt0 = extractelement <vscale x 8 x i1> %r, i32 0
+ br i1 %elt0, label %if.then, label %if.end
+
+if.then:
+ tail call void @use(<vscale x 2 x i1> %v0, <vscale x 2 x i1> %v1, <vscale x 2 x i1> %v2, <vscale x 2 x i1> %v3)
+ br label %if.end
+
+if.end:
+ ret void
+}
+
+; Negative test where the extract types are correct but we are not extracting all parts of the mask
+define void @test_partial_extract_legal_types(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_partial_extract_legal_types:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vsaddu.vx v8, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v8, a1
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vmv.v.i v8, 0
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT: srli a0, a0, 1
+; CHECK-NEXT: vslidedown.vx v8, v8, a0
+; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: tail use
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+ %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+ tail call void @use(<vscale x 2 x i1> %v0, <vscale x 2 x i1> %v1)
+ ret void
+}
+
+declare void @use(...)
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
new file mode 100644
index 0000000000000..f297fccba3a6a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
@@ -0,0 +1,836 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve %s -o - | FileCheck %s
+
+ at a = internal global ptr null
+ at b = internal global ptr null
+
+define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
+; CHECK-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: sub sp, #48
+; CHECK-NEXT: adr r2, .LCPI0_0
+; CHECK-NEXT: vdup.32 q2, r1
+; CHECK-NEXT: vldrw.u32 q0, [r2]
+; CHECK-NEXT: vmov.i8 q1, #0xff
+; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: adr r2, .LCPI0_1
+; CHECK-NEXT: vqadd.u32 q0, q0, r0
+; CHECK-NEXT: adr r3, .LCPI0_3
+; CHECK-NEXT: vcmp.u32 hi, q2, q0
+; CHECK-NEXT: vmov.i8 q0, #0x0
+; CHECK-NEXT: vpsel q3, q1, q0
+; CHECK-NEXT: vstrh.32 q3, [r1, #8]
+; CHECK-NEXT: vldrw.u32 q3, [r2]
+; CHECK-NEXT: adr r2, .LCPI0_2
+; CHECK-NEXT: vqadd.u32 q3, q3, r0
+; CHECK-NEXT: vcmp.u32 hi, q2, q3
+; CHECK-NEXT: vpsel q3, q1, q0
+; CHECK-NEXT: vstrh.32 q3, [r1]
+; CHECK-NEXT: vldrw.u32 q3, [r2]
+; CHECK-NEXT: mov r2, sp
+; CHECK-NEXT: vqadd.u32 q3, q3, r0
+; CHECK-NEXT: vcmp.u32 hi, q2, q3
+; CHECK-NEXT: vpsel q3, q1, q0
+; CHECK-NEXT: vstrh.32 q3, [r2, #8]
+; CHECK-NEXT: vldrw.u32 q3, [r3]
+; CHECK-NEXT: vqadd.u32 q3, q3, r0
+; CHECK-NEXT: add r0, sp, #32
+; CHECK-NEXT: vcmp.u32 hi, q2, q3
+; CHECK-NEXT: vpsel q2, q1, q0
+; CHECK-NEXT: vstrh.32 q2, [r2]
+; CHECK-NEXT: vldrw.u32 q2, [r1]
+; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: vcmp.i16 ne, q2, zr
+; CHECK-NEXT: vpsel q2, q1, q0
+; CHECK-NEXT: vstrb.16 q2, [r0, #8]
+; CHECK-NEXT: vldrw.u32 q2, [r2]
+; CHECK-NEXT: vcmp.i16 ne, q2, zr
+; CHECK-NEXT: vpsel q2, q1, q0
+; CHECK-NEXT: vstrb.16 q2, [r0]
+; CHECK-NEXT: vldrw.u32 q2, [r0]
+; CHECK-NEXT: vcmp.i8 ne, q2, zr
+; CHECK-NEXT: vpsel q0, q1, q0
+; CHECK-NEXT: vmov.u8 r0, q0[8]
+; CHECK-NEXT: vmov.16 q1[0], r0
+; CHECK-NEXT: vmov.u8 r0, q0[9]
+; CHECK-NEXT: vmov.16 q1[1], r0
+; CHECK-NEXT: vmov.u8 r0, q0[10]
+; CHECK-NEXT: vmov.16 q1[2], r0
+; CHECK-NEXT: vmov.u8 r0, q0[11]
+; CHECK-NEXT: vmov.16 q1[3], r0
+; CHECK-NEXT: vmov.u8 r0, q0[12]
+; CHECK-NEXT: vmov.16 q1[4], r0
+; CHECK-NEXT: vmov.u8 r0, q0[13]
+; CHECK-NEXT: vmov.16 q1[5], r0
+; CHECK-NEXT: vmov.u8 r0, q0[14]
+; CHECK-NEXT: vmov.16 q1[6], r0
+; CHECK-NEXT: vmov.u8 r0, q0[15]
+; CHECK-NEXT: vmov.16 q1[7], r0
+; CHECK-NEXT: vcmp.i16 ne, q1, zr
+; CHECK-NEXT: vmrs r2, p0
+; CHECK-NEXT: and r0, r2, #1
+; CHECK-NEXT: rsbs r3, r0, #0
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: bfi r1, r3, #0, #1
+; CHECK-NEXT: ubfx r3, r2, #2, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #1, #1
+; CHECK-NEXT: ubfx r3, r2, #4, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #2, #1
+; CHECK-NEXT: ubfx r3, r2, #6, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #3, #1
+; CHECK-NEXT: ubfx r3, r2, #8, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #4, #1
+; CHECK-NEXT: ubfx r3, r2, #10, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #5, #1
+; CHECK-NEXT: ubfx r3, r2, #12, #1
+; CHECK-NEXT: ubfx r2, r2, #14, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r1, r3, #6, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r1, r2, #7, #1
+; CHECK-NEXT: movw r2, :lower16:b
+; CHECK-NEXT: movt r2, :upper16:b
+; CHECK-NEXT: strb r1, [r2]
+; CHECK-NEXT: vmov.u8 r1, q0[0]
+; CHECK-NEXT: vmov.16 q1[0], r1
+; CHECK-NEXT: vmov.u8 r1, q0[1]
+; CHECK-NEXT: vmov.16 q1[1], r1
+; CHECK-NEXT: vmov.u8 r1, q0[2]
+; CHECK-NEXT: vmov.16 q1[2], r1
+; CHECK-NEXT: vmov.u8 r1, q0[3]
+; CHECK-NEXT: vmov.16 q1[3], r1
+; CHECK-NEXT: vmov.u8 r1, q0[4]
+; CHECK-NEXT: vmov.16 q1[4], r1
+; CHECK-NEXT: vmov.u8 r1, q0[5]
+; CHECK-NEXT: vmov.16 q1[5], r1
+; CHECK-NEXT: vmov.u8 r1, q0[6]
+; CHECK-NEXT: vmov.16 q1[6], r1
+; CHECK-NEXT: vmov.u8 r1, q0[7]
+; CHECK-NEXT: vmov.16 q1[7], r1
+; CHECK-NEXT: vcmp.i16 ne, q1, zr
+; CHECK-NEXT: vmrs r1, p0
+; CHECK-NEXT: and r2, r1, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #0, #1
+; CHECK-NEXT: ubfx r2, r1, #2, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #1, #1
+; CHECK-NEXT: ubfx r2, r1, #4, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #2, #1
+; CHECK-NEXT: ubfx r2, r1, #6, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #3, #1
+; CHECK-NEXT: ubfx r2, r1, #8, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #4, #1
+; CHECK-NEXT: ubfx r2, r1, #10, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #5, #1
+; CHECK-NEXT: ubfx r2, r1, #12, #1
+; CHECK-NEXT: ubfx r1, r1, #14, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #6, #1
+; CHECK-NEXT: rsbs r1, r1, #0
+; CHECK-NEXT: bfi r0, r1, #7, #1
+; CHECK-NEXT: movw r1, :lower16:a
+; CHECK-NEXT: movt r1, :upper16:a
+; CHECK-NEXT: strb r0, [r1]
+; CHECK-NEXT: add sp, #48
+; CHECK-NEXT: bx lr
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI0_0:
+; CHECK-NEXT: .long 12 @ 0xc
+; CHECK-NEXT: .long 13 @ 0xd
+; CHECK-NEXT: .long 14 @ 0xe
+; CHECK-NEXT: .long 15 @ 0xf
+; CHECK-NEXT: .LCPI0_1:
+; CHECK-NEXT: .long 8 @ 0x8
+; CHECK-NEXT: .long 9 @ 0x9
+; CHECK-NEXT: .long 10 @ 0xa
+; CHECK-NEXT: .long 11 @ 0xb
+; CHECK-NEXT: .LCPI0_2:
+; CHECK-NEXT: .long 4 @ 0x4
+; CHECK-NEXT: .long 5 @ 0x5
+; CHECK-NEXT: .long 6 @ 0x6
+; CHECK-NEXT: .long 7 @ 0x7
+; CHECK-NEXT: .LCPI0_3:
+; CHECK-NEXT: .long 0 @ 0x0
+; CHECK-NEXT: .long 1 @ 0x1
+; CHECK-NEXT: .long 2 @ 0x2
+; CHECK-NEXT: .long 3 @ 0x3
+ %r = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %i, i32 %n)
+ %v0 = call <8 x i1> @llvm.vector.extract.v8i1.v16i1.i64(<16 x i1> %r, i64 0)
+ %v1 = call <8 x i1> @llvm.vector.extract.v8i1.v16i1.i64(<16 x i1> %r, i64 8)
+ store <8 x i1> %v0, ptr @a
+ store <8 x i1> %v1, ptr @b
+ ret void
+}
+
+define void @test_2x16bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x16bit_mask_with_64bit_index_and_trip_count:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT: adds.w r12, r0, #17
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r4, r12, r0
+; CHECK-NEXT: sbcs.w r4, lr, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r5, r0, #16
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r5, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r5, r12
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r4
+; CHECK-NEXT: mov.w lr, #0
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r4
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #0, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #19
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #1, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #18
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #2, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #21
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #3, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #20
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #4, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #23
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #5, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #22
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #6, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #25
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #7, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #24
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #8, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #27
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #9, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #26
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #10, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #29
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #11, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #28
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #12, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds.w r9, r0, #31
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: bfi lr, r4, #13, #1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds.w r7, r0, #30
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vldr s4, .LCPI1_0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov.f32 s5, s4
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: bfi lr, r4, #14, #1
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds r7, r0, #1
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r7
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r7, r0
+; CHECK-NEXT: vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: bfi lr, r4, #15, #1
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: movw r4, :lower16:b
+; CHECK-NEXT: vmov s6, r7
+; CHECK-NEXT: movt r4, :upper16:b
+; CHECK-NEXT: strh.w lr, [r4]
+; CHECK-NEXT: vmov.f32 s7, s6
+; CHECK-NEXT: vorr q0, q0, q1
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #0, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #3
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #1, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds r4, r0, #2
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #2, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #5
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #3, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds r4, r0, #4
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #4, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #7
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #5, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds r4, r0, #6
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #6, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #9
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #7, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds.w r4, r0, #8
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #8, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #11
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #9, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds.w r4, r0, #10
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #10, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r8, r0, #13
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r5, r8, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: bfi r12, r7, #11, #1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds.w r4, r0, #12
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #12, #1
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r12, r7, #13, #1
+; CHECK-NEXT: adds.w r7, r0, #15
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs r5, r7, r0
+; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds.w r4, r0, #14
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r0, r4, r0
+; CHECK-NEXT: sbcs.w r0, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r7
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r0, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r0, r1, d0
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: bfi r12, r0, #14, #1
+; CHECK-NEXT: vmov r0, r1, d1
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: bfi r12, r0, #15, #1
+; CHECK-NEXT: movw r0, :lower16:a
+; CHECK-NEXT: movt r0, :upper16:a
+; CHECK-NEXT: strh.w r12, [r0]
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI1_0:
+; CHECK-NEXT: .long 0x00000000 @ float 0
+ %r = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i64 %i, i64 %n)
+ %v0 = call <16 x i1> @llvm.vector.extract.v16i1.v32i1.i64(<32 x i1> %r, i64 0)
+ %v1 = call <16 x i1> @llvm.vector.extract.v16i1.v32i1.i64(<32 x i1> %r, i64 16)
+ store <16 x i1> %v0, ptr @a
+ store <16 x i1> %v1, ptr @b
+ ret void
+}
+
+; Negative test for when not extracting exactly two halves of the source vector
+define void @test_partial_extract(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_partial_extract:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: push {r4, r5, r6, r7, lr}
+; CHECK-NEXT: adds.w r12, r0, #5
+; CHECK-NEXT: adc lr, r1, #0
+; CHECK-NEXT: subs.w r4, r12, r0
+; CHECK-NEXT: sbcs.w r4, lr, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: adds r5, r0, #4
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r5, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r5, r12
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r4
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r4
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vldr s4, .LCPI2_0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: vmov.f32 s5, s4
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: mov.w r6, #0
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r6, r7, #0, #1
+; CHECK-NEXT: vmov r7, r5, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r5, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: bfi r6, r7, #1, #1
+; CHECK-NEXT: movw r7, :lower16:b
+; CHECK-NEXT: movt r7, :upper16:b
+; CHECK-NEXT: strb r6, [r7]
+; CHECK-NEXT: adds r7, r0, #1
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r7
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r0, r7, r0
+; CHECK-NEXT: vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT: sbcs.w r0, r6, r1
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: vmov s6, r0
+; CHECK-NEXT: vmov.f32 s7, s6
+; CHECK-NEXT: vorr q0, q0, q1
+; CHECK-NEXT: vmov r0, r1, d0
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: bfi r12, r0, #0, #1
+; CHECK-NEXT: vmov r0, r1, d1
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: bfi r12, r0, #1, #1
+; CHECK-NEXT: movw r0, :lower16:a
+; CHECK-NEXT: movt r0, :upper16:a
+; CHECK-NEXT: strb.w r12, [r0]
+; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI2_0:
+; CHECK-NEXT: .long 0x00000000 @ float 0
+ %r = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 %i, i64 %n)
+ %v0 = call <2 x i1> @llvm.vector.extract.nxv2i1.v8i1.i64(<8 x i1> %r, i64 0)
+ %v1 = call <2 x i1> @llvm.vector.extract.nxv2i1.v8i1.i64(<8 x i1> %r, i64 4)
+ store <2 x i1> %v0, ptr @a
+ store <2 x i1> %v1, ptr @b
+ ret void
+}
+
+define void @test_2x4bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x4bit_mask_with_extracts_and_reinterpret_casts:
+; CHECK: @ %bb.0: @ %entry
+; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT: sub sp, #4
+; CHECK-NEXT: adds.w lr, r0, #5
+; CHECK-NEXT: adc r12, r1, #0
+; CHECK-NEXT: subs.w r5, lr, r0
+; CHECK-NEXT: sbcs.w r5, r12, r1
+; CHECK-NEXT: csetm r5, lo
+; CHECK-NEXT: adds r4, r0, #4
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r6, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r4, lr
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r6, r12
+; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: str r7, [sp] @ 4-byte Spill
+; CHECK-NEXT: vmov r7, r6, d1
+; CHECK-NEXT: subs r7, r7, r2
+; CHECK-NEXT: sbcs.w r7, r6, r3
+; CHECK-NEXT: csetm lr, lo
+; CHECK-NEXT: adds.w r9, r0, #7
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r6, r9, r0
+; CHECK-NEXT: sbcs.w r6, r8, r1
+; CHECK-NEXT: csetm r6, lo
+; CHECK-NEXT: adds r7, r0, #6
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: sbcs.w r4, r5, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vldr s4, .LCPI3_0
+; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov.f32 s5, s4
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: csetm r8, lo
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r9, lo
+; CHECK-NEXT: adds r4, r0, #1
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT: adc r6, r1, #0
+; CHECK-NEXT: subs r4, r4, r0
+; CHECK-NEXT: vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT: sbcs.w r4, r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov s6, r4
+; CHECK-NEXT: vmov.f32 s7, s6
+; CHECK-NEXT: vorr q0, q0, q1
+; CHECK-NEXT: vmov r4, r6, d0
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r6, r3
+; CHECK-NEXT: vmov r4, r7, d1
+; CHECK-NEXT: cset r11, lo
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r7, r3
+; CHECK-NEXT: csetm r10, lo
+; CHECK-NEXT: adds r4, r0, #3
+; CHECK-NEXT: adc r5, r1, #0
+; CHECK-NEXT: subs r7, r4, r0
+; CHECK-NEXT: sbcs.w r7, r5, r1
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds r6, r0, #2
+; CHECK-NEXT: adc r12, r1, #0
+; CHECK-NEXT: subs r0, r6, r0
+; CHECK-NEXT: sbcs.w r0, r12, r1
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r4
+; CHECK-NEXT: csetm r0, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r12, r5
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r7
+; CHECK-NEXT: vmov q0[3], q0[1], r0, r7
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r0, r1, d0
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: vmov r0, r1, d1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: subs r0, r0, r2
+; CHECK-NEXT: sbcs.w r0, r1, r3
+; CHECK-NEXT: csetm r3, lo
+; CHECK-NEXT: cmp.w r11, #0
+; CHECK-NEXT: beq .LBB3_2
+; CHECK-NEXT: @ %bb.1: @ %if.then
+; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: rsb.w r7, r11, #0
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: bfi r2, r1, #0, #4
+; CHECK-NEXT: movs r1, #0
+; CHECK-NEXT: bfi r1, r7, #0, #4
+; CHECK-NEXT: bfi r2, lr, #4, #4
+; CHECK-NEXT: bfi r2, r8, #8, #4
+; CHECK-NEXT: bfi r1, r10, #4, #4
+; CHECK-NEXT: bfi r1, r4, #8, #4
+; CHECK-NEXT: bfi r2, r9, #12, #4
+; CHECK-NEXT: bfi r1, r3, #12, #4
+; CHECK-NEXT: and r3, r2, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: movs r7, #0
+; CHECK-NEXT: bfi r7, r3, #0, #1
+; CHECK-NEXT: ubfx r3, r2, #4, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r7, r3, #1, #1
+; CHECK-NEXT: ubfx r3, r2, #8, #1
+; CHECK-NEXT: ubfx r2, r2, #12, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r7, r3, #2, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r7, r2, #3, #1
+; CHECK-NEXT: movw r2, :lower16:b
+; CHECK-NEXT: movt r2, :upper16:b
+; CHECK-NEXT: strb r7, [r2]
+; CHECK-NEXT: and r2, r1, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #0, #1
+; CHECK-NEXT: ubfx r2, r1, #4, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #1, #1
+; CHECK-NEXT: ubfx r2, r1, #8, #1
+; CHECK-NEXT: ubfx r1, r1, #12, #1
+; CHECK-NEXT: rsbs r2, r2, #0
+; CHECK-NEXT: bfi r0, r2, #2, #1
+; CHECK-NEXT: rsbs r1, r1, #0
+; CHECK-NEXT: bfi r0, r1, #3, #1
+; CHECK-NEXT: movw r1, :lower16:a
+; CHECK-NEXT: movt r1, :upper16:a
+; CHECK-NEXT: strb r0, [r1]
+; CHECK-NEXT: .LBB3_2: @ %if.end
+; CHECK-NEXT: add sp, #4
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: @ %bb.3:
+; CHECK-NEXT: .LCPI3_0:
+; CHECK-NEXT: .long 0x00000000 @ float 0
+entry:
+ %r = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 %i, i64 %n)
+ %v0 = tail call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> %r, i64 0)
+ %v1 = tail call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> %r, i64 4)
+ %elt0 = extractelement <8 x i1> %r, i64 0
+ br i1 %elt0, label %if.then, label %if.end
+
+if.then:
+ store <4 x i1> %v0, ptr @a
+ store <4 x i1> %v1, ptr @b
+ br label %if.end
+
+if.end:
+ ret void
+}
+
+attributes #0 = { nounwind }
>From 1daa0aa9955b0b72799c7fa36a9d1d363e8f1ea0 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 8 Jun 2026 09:56:53 +0000
Subject: [PATCH 2/2] [CodeGen] Split wide active lane mask into smaller masks
if preferred
Adds optimizeGetActiveLaneMask to CodeGenPrepare, which will rewrite a
get.active.lane.mask used by a series of subvector extracts if this is the
form preferred by the target.
This PR is part of a series of changes leading to the use of wide active
lane masks as the canonical form in LoopVectorize when choosing to interleave
and tail-fold loops. https://github.com/llvm/llvm-project/pull/193757 is the
original PR where this was discussed.
For all targets other than AArch64, the preference is to use multiple smaller
get.active.lane.mask intrinsics, which matches the output of LoopVectorize today
when interleaving, unless wide active lane masks are forced. For AArch64,
prefer a wide mask + extracts when either SVE2p1 or SME2 is available.
The motivation for this PR is to leave the decision on which form of the IR
to use up to the target. This will remove the need for more complex decisions
and transformations during vectorisation to query whether it's best to emit
a single mask + extracts or multiple smaller masks.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 +
llvm/lib/CodeGen/CodeGenPrepare.cpp | 100 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +
.../AArch64/get-active-lane-mask-extract.ll | 170 +++-
.../RISCV/rvv/get-active-lane-mask-extract.ll | 96 +-
.../Thumb2/get-active-lane-mask-extract.ll | 940 ++++++++----------
7 files changed, 719 insertions(+), 596 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 82c47cce0f522..288d3c31521b1 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -487,6 +487,8 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
+ virtual bool preferWideActiveLaneMask() const { return false; }
+
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 047ca6cebb1c6..82a7da2d51bd9 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -426,6 +426,7 @@ class CodeGenPrepare {
bool optimizeGatherScatterInst(Instruction *MemoryInst, Value *Ptr);
bool optimizeMulWithOverflow(Instruction *I, bool IsSigned,
ModifyDT &ModifiedDT);
+ bool optimizeGetActiveLaneMask(Instruction *I);
bool optimizeInlineAsmInst(CallInst *CS);
bool optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT);
bool optimizeExt(Instruction *&I);
@@ -2834,6 +2835,8 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
return optimizeMulWithOverflow(II, /*IsSigned=*/false, ModifiedDT);
case Intrinsic::smul_with_overflow:
return optimizeMulWithOverflow(II, /*IsSigned=*/true, ModifiedDT);
+ case Intrinsic::get_active_lane_mask:
+ return optimizeGetActiveLaneMask(II);
}
SmallVector<Value *, 2> PtrOps;
@@ -6608,6 +6611,103 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
return true;
}
+// Rewrite a get.active.lane.mask intrinsic followed by vector extracts
+// into multiple smaller get.active.lane.mask intrinsics. This should
+// only be rewritten if the target prefers this form over using a
+// wide active lane mask.
+//
+// Example:
+// %alm = call <16 x i1> @llvm.get.active.lane.mask(i32 %i, i32 %n)
+// %p0 = call <8 x i1> @llvm.vector.extract(<16 x i1> %alm, i64 0)
+// %p1 = call <8 x i1> @llvm.vector.extract(<16 x i1> %alm, i64 8)
+//
+// is rewritten as:
+// %alm0 = call <8 x i1> @llvm.get.active.lane.mask(i32 %i, i32 %n)
+// %next = call i32 @llvm.uadd.sat(i32 %i, i32 8)
+// %alm1 = call <8 x i1> @llvm.get.active.lane.mask(i32 %next, i32 %n)
+//
+bool CodeGenPrepare::optimizeGetActiveLaneMask(Instruction *ALM) {
+ if (TLI->preferWideActiveLaneMask())
+ return false;
+
+ // Count the number of users of ALM which are vector extracts
+ unsigned NumExts = count_if(ALM->users(), [](User *U) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ return II && II->getIntrinsicID() == Intrinsic::vector_extract;
+ });
+
+ // Return false if the extracts are not the right type for the number
+ // of parts (e.g. 4 x v2i1 extracts of a single v8i1 mask).
+ auto RetTy = cast<VectorType>(ALM->getType());
+ auto MaskEC = RetTy->getElementCount();
+ if (NumExts < 2 || !MaskEC.isKnownMultipleOf(NumExts))
+ return false;
+
+ ElementCount ExtractEC = MaskEC.divideCoefficientBy(NumExts);
+ if (ExtractEC.getKnownMinValue() < 2)
+ return false;
+
+ SmallVector<IntrinsicInst *> Extracts(NumExts, nullptr);
+ for (User *U : ALM->users()) {
+ // The only users accepted are vector extracts and an extract of
+ // element 0.
+ if (auto ExtElt = dyn_cast<ExtractElementInst>(U)) {
+ auto CIdx = dyn_cast<ConstantInt>(ExtElt->getIndexOperand());
+ if (!CIdx || CIdx->getZExtValue() != 0)
+ return false;
+ continue;
+ }
+
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II || II->getIntrinsicID() != Intrinsic::vector_extract)
+ return false;
+
+ unsigned Idx = cast<ConstantInt>(II->getOperand(1))->getZExtValue();
+ if (cast<VectorType>(II->getType())->getElementCount() != ExtractEC)
+ return false;
+
+ unsigned Part = Idx / ExtractEC.getKnownMinValue();
+ if (Extracts[Part] != nullptr)
+ return false;
+
+ Extracts[Part] = II;
+ }
+
+ IRBuilder<> Builder(ALM);
+ Value *Part0Mask;
+ auto *StartVal = ALM->getOperand(0);
+ auto StartEC = Builder.CreateElementCount(StartVal->getType(), ExtractEC);
+ // Replace each vector extract with a get.active.lane.mask of the same type.
+ for (unsigned I = 0; I < NumExts; ++I) {
+ auto ExtTy = Extracts[0]->getType();
+ if (I > 0)
+ StartVal =
+ Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, StartVal, StartEC);
+
+ Value *ALMForPart = Builder.CreateIntrinsic(
+ ExtTy, Intrinsic::get_active_lane_mask, {StartVal, ALM->getOperand(1)});
+ if (I == 0)
+ Part0Mask = ALMForPart;
+ Extracts[I]->replaceAllUsesWith(ALMForPart);
+ Extracts[I]->replaceUsesOfWith(ALM, PoisonValue::get(RetTy));
+ }
+
+ // Update the remaining users, which must be extracts of element 0 from
+ // the original wide mask.
+ SmallVector<User *> RemainingUsers(ALM->users());
+ for (User *U : RemainingUsers) {
+ auto *ExtElt = dyn_cast<ExtractElementInst>(U);
+ assert(ExtElt && "Unexpected user of get.active.lane.mask");
+ auto *NewExt =
+ Builder.CreateExtractElement(Part0Mask, ExtElt->getIndexOperand());
+ ExtElt->replaceAllUsesWith(NewExt);
+ ExtElt->replaceUsesOfWith(ALM, PoisonValue::get(RetTy));
+ }
+ ALM->eraseFromParent();
+
+ return true;
+}
+
/// If there are any memory operands, use OptimizeMemoryInst to sink their
/// address computing into the block when possible / profitable.
bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 08f3d4e0d30ac..c75ad4ae3102d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2364,6 +2364,11 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
}
}
+bool AArch64TargetLowering::preferWideActiveLaneMask() const {
+ return Subtarget->isSVEorStreamingSVEAvailable() &&
+ (Subtarget->hasSVE2p1() || Subtarget->hasSME2());
+}
+
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
EVT OpVT) const {
// Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 3a93d9a3c0d1e..1f5f0079397ce 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -551,6 +551,8 @@ class AArch64TargetLowering : public TargetLowering {
EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
bool AllowUnknown = false) const override;
+ bool preferWideActiveLaneMask() const override;
+
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
bool shouldExpandCttzElements(EVT VT) const override;
diff --git a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
index 62cf5d686fd97..82c1b08c67a13 100644
--- a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
+++ b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
@@ -10,9 +10,11 @@ target triple = "aarch64-linux"
define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.b, w0, w1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, w0, w1
+; CHECK-SVE-NEXT: adds w8, w0, w8
+; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT: whilelo p1.h, w8, w1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
@@ -31,9 +33,11 @@ define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0
define void @test_2x8bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.b, x0, x1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.h, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
@@ -71,9 +75,11 @@ define void @test_edge_case_2x1bit_mask(i64 %i, i64 %n) #0 {
define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_boring_case_2x2bit_mask:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.s, x0, x1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cntd x8
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.d, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_boring_case_2x2bit_mask:
@@ -90,13 +96,17 @@ define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
define void @test_legal_4x2bit_mask(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_legal_4x2bit_mask:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
-; CHECK-SVE-NEXT: punpkhi p1.h, p0.b
-; CHECK-SVE-NEXT: punpklo p4.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p1.b
-; CHECK-SVE-NEXT: punpklo p0.h, p4.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p4.b
+; CHECK-SVE-NEXT: cntd x8
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: adds x9, x0, x8
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.d, x9, x1
+; CHECK-SVE-NEXT: adds x9, x9, x8
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: whilelo p2.d, x9, x1
+; CHECK-SVE-NEXT: adds x8, x9, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p3.d, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_legal_4x2bit_mask:
@@ -250,9 +260,9 @@ define void @test_2x16bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
; CHECK-SVE-LABEL: test_2x16bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
; CHECK-SVE-NEXT: rdvl x8, #1
+; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
; CHECK-SVE-NEXT: adds w8, w0, w8
; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
-; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
; CHECK-SVE-NEXT: whilelo p1.b, w8, w1
; CHECK-SVE-NEXT: b use
;
@@ -272,18 +282,17 @@ define void @test_2x16bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
define void @test_2x32bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
; CHECK-SVE-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: rdvl x8, #2
-; CHECK-SVE-NEXT: rdvl x9, #1
-; CHECK-SVE-NEXT: adds w8, w0, w8
-; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
-; CHECK-SVE-NEXT: adds w10, w8, w9
-; CHECK-SVE-NEXT: csinv w10, w10, wzr, lo
-; CHECK-SVE-NEXT: whilelo p3.b, w10, w1
-; CHECK-SVE-NEXT: adds w9, w0, w9
-; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
+; CHECK-SVE-NEXT: rdvl x8, #1
; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
+; CHECK-SVE-NEXT: adds w9, w0, w8
+; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
; CHECK-SVE-NEXT: whilelo p1.b, w9, w1
-; CHECK-SVE-NEXT: whilelo p2.b, w8, w1
+; CHECK-SVE-NEXT: adds w9, w9, w8
+; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
+; CHECK-SVE-NEXT: whilelo p2.b, w9, w1
+; CHECK-SVE-NEXT: adds w8, w9, w8
+; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT: whilelo p3.b, w8, w1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
@@ -310,11 +319,14 @@ define void @test_2x32bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
define void @test_2x8bit_mask_with_extracts_and_ptest(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_extracts_and_ptest:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p1.b, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB11_2
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT: cset w9, mi
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: cbz w9, .LBB11_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: whilelo p1.h, x8, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB11_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -348,11 +360,14 @@ if.end:
define void @test_2x8bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_extracts_and_reinterpret_casts:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p1.h, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB12_2
+; CHECK-SVE-NEXT: cntw x8
+; CHECK-SVE-NEXT: whilelo p0.s, x0, x1
+; CHECK-SVE-NEXT: cset w9, mi
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: cbz w9, .LBB12_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: whilelo p1.s, x8, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB12_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -383,15 +398,20 @@ if.end:
define void @test_4x4bit_mask_with_extracts_and_ptest(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_4x4bit_mask_with_extracts_and_ptest:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p0.b, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB13_2
+; CHECK-SVE-NEXT: cntw x10
+; CHECK-SVE-NEXT: whilelo p0.s, x0, x1
+; CHECK-SVE-NEXT: cset w11, mi
+; CHECK-SVE-NEXT: adds x8, x0, x10
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: adds x9, x8, x10
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: adds x10, x9, x10
+; CHECK-SVE-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SVE-NEXT: cbz w11, .LBB13_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p1.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p0.b
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p3.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p3.b
+; CHECK-SVE-NEXT: whilelo p1.s, x8, x1
+; CHECK-SVE-NEXT: whilelo p2.s, x9, x1
+; CHECK-SVE-NEXT: whilelo p3.s, x10, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB13_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -428,15 +448,20 @@ if.end:
define void @test_4x2bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_4x2bit_mask_with_extracts_and_reinterpret_casts:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB14_2
+; CHECK-SVE-NEXT: cntd x10
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: cset w11, mi
+; CHECK-SVE-NEXT: adds x8, x0, x10
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: adds x9, x8, x10
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: adds x10, x9, x10
+; CHECK-SVE-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SVE-NEXT: cbz w11, .LBB14_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p1.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p0.b
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p3.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p3.b
+; CHECK-SVE-NEXT: whilelo p1.d, x8, x1
+; CHECK-SVE-NEXT: whilelo p2.d, x9, x1
+; CHECK-SVE-NEXT: whilelo p3.d, x10, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB14_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -470,6 +495,49 @@ if.end:
ret void
}
+; Test use of get.active.lane.mask by an extractelement with an index value other than 0.
+define void @test_2x8bit_mask_with_invalid_extract_elt(i64 %i, i64 %n) {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_invalid_extract_elt:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: whilelo p1.h, x0, x1
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-SVE-NEXT: mov z0.h, z0.h[8]
+; CHECK-SVE-NEXT: fmov w8, s0
+; CHECK-SVE-NEXT: tbz w8, #0, .LBB15_2
+; CHECK-SVE-NEXT: // %bb.1: // %if.then
+; CHECK-SVE-NEXT: punpklo p0.h, p1.b
+; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: b use
+; CHECK-SVE-NEXT: .LBB15_2: // %if.end
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_invalid_extract_elt:
+; CHECK-SVE2p1-SME2: // %bb.0: // %entry
+; CHECK-SVE2p1-SME2-NEXT: whilelo { p0.s, p1.s }, x0, x1
+; CHECK-SVE2p1-SME2-NEXT: uzp1 p2.h, p0.h, p1.h
+; CHECK-SVE2p1-SME2-NEXT: mov z0.h, p2/z, #1 // =0x1
+; CHECK-SVE2p1-SME2-NEXT: mov z0.h, z0.h[8]
+; CHECK-SVE2p1-SME2-NEXT: fmov w8, s0
+; CHECK-SVE2p1-SME2-NEXT: tbz w8, #0, .LBB15_2
+; CHECK-SVE2p1-SME2-NEXT: // %bb.1: // %if.then
+; CHECK-SVE2p1-SME2-NEXT: b use
+; CHECK-SVE2p1-SME2-NEXT: .LBB15_2: // %if.end
+; CHECK-SVE2p1-SME2-NEXT: ret
+entry:
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+ %v0 = tail call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> %r, i64 0)
+ %v1 = tail call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> %r, i64 4)
+ %elt0 = extractelement <vscale x 8 x i1> %r, i64 8
+ br i1 %elt0, label %if.then, label %if.end
+
+if.then:
+ tail call void @use(<vscale x 4 x i1> %v0, <vscale x 4 x i1> %v1)
+ br label %if.end
+
+if.end:
+ ret void
+}
+
declare void @use(...)
attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
index e6a514645ee28..24b9e6d13e44b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
@@ -4,18 +4,18 @@
define void @test_2x4bit_mask_with_extracts(i64 %i, i64 %n) #0 {
; CHECK-LABEL: test_2x4bit_mask_with_extracts:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: vsetvli a3, zero, e64, m4, ta, ma
; CHECK-NEXT: vid.v v8
-; CHECK-NEXT: vsaddu.vx v8, v8, a0
-; CHECK-NEXT: vmsltu.vx v0, v8, a1
-; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT: srli a0, a0, 1
-; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma
-; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: srli a2, a2, 1
+; CHECK-NEXT: vsaddu.vx v12, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v12, a1
+; CHECK-NEXT: add a2, a0, a2
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: vsaddu.vx v12, v8, a0
+; CHECK-NEXT: vmsltu.vx v8, v12, a1
; CHECK-NEXT: tail use
%r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
%v0 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
@@ -27,12 +27,15 @@ define void @test_2x4bit_mask_with_extracts(i64 %i, i64 %n) #0 {
define void @test_2x8bit_mask_with_extracts(i64 %i, i64 %n) #0 {
; CHECK-LABEL: test_2x8bit_mask_with_extracts:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
-; CHECK-NEXT: vid.v v8
; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: vsetvli a3, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vsaddu.vx v16, v8, a0
-; CHECK-NEXT: vadd.vx v8, v8, a2
+; CHECK-NEXT: add a2, a0, a2
; CHECK-NEXT: vmsltu.vx v0, v16, a1
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
; CHECK-NEXT: vsaddu.vx v16, v8, a0
; CHECK-NEXT: vmsltu.vx v8, v16, a1
; CHECK-NEXT: tail use
@@ -46,17 +49,16 @@ define void @test_2x8bit_mask_with_extracts(i64 %i, i64 %n) #0 {
define void @test_fixed_2x4bit_mask_with_extract(i64 %i, i64 %n) #0 {
; CHECK-LABEL: test_fixed_2x4bit_mask_with_extract:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vid.v v8
-; CHECK-NEXT: vsaddu.vx v8, v8, a0
-; CHECK-NEXT: vmsltu.vx v0, v8, a1
-; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma
-; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT: vsetivli zero, 4, e8, mf2, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 4
-; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT: vmsne.vi v8, v8, 0
+; CHECK-NEXT: addi a2, a0, 4
+; CHECK-NEXT: vsaddu.vx v10, v8, a0
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: vmsltu.vx v0, v10, a1
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: vsaddu.vx v10, v8, a0
+; CHECK-NEXT: vmsltu.vx v8, v10, a1
; CHECK-NEXT: tail use
%r = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 %i, i64 %n)
%v0 = call <4 x i1> @llvm.vector.extract.v4i1.v8i1.i64(<8 x i1> %r, i64 0)
@@ -68,29 +70,33 @@ define void @test_fixed_2x4bit_mask_with_extract(i64 %i, i64 %n) #0 {
define void @test_4x2bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) #0 {
; CHECK-LABEL: test_4x2bit_mask_with_extracts_and_reinterpret_casts:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetvli a2, zero, e64, m8, ta, ma
-; CHECK-NEXT: vid.v v8
-; CHECK-NEXT: vsaddu.vx v8, v8, a0
+; CHECK-NEXT: vsetvli a2, zero, e64, m2, ta, ma
+; CHECK-NEXT: vid.v v10
+; CHECK-NEXT: vsaddu.vx v8, v10, a0
; CHECK-NEXT: vmsltu.vx v0, v8, a1
-; CHECK-NEXT: vfirst.m a0, v0
-; CHECK-NEXT: bnez a0, .LBB3_2
+; CHECK-NEXT: vfirst.m a2, v0
+; CHECK-NEXT: bnez a2, .LBB3_2
; CHECK-NEXT: # %bb.1: # %if.then
-; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vmerge.vim v10, v8, 1, v0
-; CHECK-NEXT: srli a1, a0, 2
-; CHECK-NEXT: srli a2, a0, 1
-; CHECK-NEXT: vslidedown.vx v8, v10, a1
-; CHECK-NEXT: vslidedown.vx v9, v10, a2
-; CHECK-NEXT: sub a0, a0, a1
-; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma
-; CHECK-NEXT: vmsne.vi v8, v8, 0
-; CHECK-NEXT: vmsne.vi v9, v9, 0
-; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vx v10, v10, a0
-; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma
-; CHECK-NEXT: vmsne.vi v10, v10, 0
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: srli a2, a2, 2
+; CHECK-NEXT: add a3, a0, a2
+; CHECK-NEXT: sltu a0, a3, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a3
+; CHECK-NEXT: vsaddu.vx v12, v10, a0
+; CHECK-NEXT: add a3, a0, a2
+; CHECK-NEXT: vmsltu.vx v8, v12, a1
+; CHECK-NEXT: sltu a0, a3, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a3
+; CHECK-NEXT: vsaddu.vx v12, v10, a0
+; CHECK-NEXT: add a2, a0, a2
+; CHECK-NEXT: vmsltu.vx v9, v12, a1
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: vsaddu.vx v12, v10, a0
+; CHECK-NEXT: vmsltu.vx v10, v12, a1
; CHECK-NEXT: tail use
; CHECK-NEXT: .LBB3_2: # %if.end
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
index f297fccba3a6a..221a6a3847f96 100644
--- a/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
+++ b/llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll
@@ -7,164 +7,96 @@
define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
; CHECK-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
; CHECK: @ %bb.0:
-; CHECK-NEXT: sub sp, #48
+; CHECK-NEXT: push {r7, lr}
; CHECK-NEXT: adr r2, .LCPI0_0
-; CHECK-NEXT: vdup.32 q2, r1
-; CHECK-NEXT: vldrw.u32 q0, [r2]
-; CHECK-NEXT: vmov.i8 q1, #0xff
-; CHECK-NEXT: add r1, sp, #16
-; CHECK-NEXT: adr r2, .LCPI0_1
-; CHECK-NEXT: vqadd.u32 q0, q0, r0
-; CHECK-NEXT: adr r3, .LCPI0_3
-; CHECK-NEXT: vcmp.u32 hi, q2, q0
-; CHECK-NEXT: vmov.i8 q0, #0x0
-; CHECK-NEXT: vpsel q3, q1, q0
-; CHECK-NEXT: vstrh.32 q3, [r1, #8]
-; CHECK-NEXT: vldrw.u32 q3, [r2]
-; CHECK-NEXT: adr r2, .LCPI0_2
-; CHECK-NEXT: vqadd.u32 q3, q3, r0
-; CHECK-NEXT: vcmp.u32 hi, q2, q3
-; CHECK-NEXT: vpsel q3, q1, q0
-; CHECK-NEXT: vstrh.32 q3, [r1]
-; CHECK-NEXT: vldrw.u32 q3, [r2]
-; CHECK-NEXT: mov r2, sp
-; CHECK-NEXT: vqadd.u32 q3, q3, r0
-; CHECK-NEXT: vcmp.u32 hi, q2, q3
-; CHECK-NEXT: vpsel q3, q1, q0
-; CHECK-NEXT: vstrh.32 q3, [r2, #8]
-; CHECK-NEXT: vldrw.u32 q3, [r3]
-; CHECK-NEXT: vqadd.u32 q3, q3, r0
-; CHECK-NEXT: add r0, sp, #32
-; CHECK-NEXT: vcmp.u32 hi, q2, q3
-; CHECK-NEXT: vpsel q2, q1, q0
-; CHECK-NEXT: vstrh.32 q2, [r2]
-; CHECK-NEXT: vldrw.u32 q2, [r1]
+; CHECK-NEXT: vdup.32 q0, r1
+; CHECK-NEXT: vldrw.u32 q1, [r2]
+; CHECK-NEXT: movs r2, #0
+; CHECK-NEXT: vqadd.u32 q2, q1, r0
+; CHECK-NEXT: vcmp.u32 hi, q0, q2
+; CHECK-NEXT: vmrs lr, p0
+; CHECK-NEXT: and r1, lr, #1
+; CHECK-NEXT: rsb.w r12, r1, #0
+; CHECK-NEXT: ubfx r3, lr, #4, #1
+; CHECK-NEXT: bfi r2, r12, #0, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #1, #1
+; CHECK-NEXT: ubfx r3, lr, #8, #1
+; CHECK-NEXT: rsbs r3, r3, #0
; CHECK-NEXT: movs r1, #0
-; CHECK-NEXT: vcmp.i16 ne, q2, zr
-; CHECK-NEXT: vpsel q2, q1, q0
-; CHECK-NEXT: vstrb.16 q2, [r0, #8]
-; CHECK-NEXT: vldrw.u32 q2, [r2]
-; CHECK-NEXT: vcmp.i16 ne, q2, zr
-; CHECK-NEXT: vpsel q2, q1, q0
-; CHECK-NEXT: vstrb.16 q2, [r0]
-; CHECK-NEXT: vldrw.u32 q2, [r0]
-; CHECK-NEXT: vcmp.i8 ne, q2, zr
-; CHECK-NEXT: vpsel q0, q1, q0
-; CHECK-NEXT: vmov.u8 r0, q0[8]
-; CHECK-NEXT: vmov.16 q1[0], r0
-; CHECK-NEXT: vmov.u8 r0, q0[9]
-; CHECK-NEXT: vmov.16 q1[1], r0
-; CHECK-NEXT: vmov.u8 r0, q0[10]
-; CHECK-NEXT: vmov.16 q1[2], r0
-; CHECK-NEXT: vmov.u8 r0, q0[11]
-; CHECK-NEXT: vmov.16 q1[3], r0
-; CHECK-NEXT: vmov.u8 r0, q0[12]
-; CHECK-NEXT: vmov.16 q1[4], r0
-; CHECK-NEXT: vmov.u8 r0, q0[13]
-; CHECK-NEXT: vmov.16 q1[5], r0
-; CHECK-NEXT: vmov.u8 r0, q0[14]
-; CHECK-NEXT: vmov.16 q1[6], r0
-; CHECK-NEXT: vmov.u8 r0, q0[15]
-; CHECK-NEXT: vmov.16 q1[7], r0
-; CHECK-NEXT: vcmp.i16 ne, q1, zr
+; CHECK-NEXT: bfi r2, r3, #2, #1
+; CHECK-NEXT: ubfx r3, lr, #12, #1
+; CHECK-NEXT: rsbs r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #3, #1
+; CHECK-NEXT: adr r3, .LCPI0_1
+; CHECK-NEXT: vldrw.u32 q2, [r3]
+; CHECK-NEXT: vqadd.u32 q3, q2, r0
+; CHECK-NEXT: adds r0, #8
+; CHECK-NEXT: vcmp.u32 hi, q0, q3
+; CHECK-NEXT: vmrs r12, p0
+; CHECK-NEXT: and r3, r12, #1
+; CHECK-NEXT: rsb.w r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #4, #1
+; CHECK-NEXT: ubfx r3, r12, #4, #1
+; CHECK-NEXT: rsb.w r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #5, #1
+; CHECK-NEXT: ubfx r3, r12, #8, #1
+; CHECK-NEXT: rsb.w r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #6, #1
+; CHECK-NEXT: ubfx r3, r12, #12, #1
+; CHECK-NEXT: rsb.w r3, r3, #0
+; CHECK-NEXT: bfi r2, r3, #7, #1
+; CHECK-NEXT: movw r3, :lower16:a
+; CHECK-NEXT: movt r3, :upper16:a
+; CHECK-NEXT: strb r2, [r3]
+; CHECK-NEXT: it hs
+; CHECK-NEXT: movhs.w r0, #-1
+; CHECK-NEXT: vqadd.u32 q1, q1, r0
+; CHECK-NEXT: vcmp.u32 hi, q0, q1
+; CHECK-NEXT: vqadd.u32 q1, q2, r0
; CHECK-NEXT: vmrs r2, p0
-; CHECK-NEXT: and r0, r2, #1
-; CHECK-NEXT: rsbs r3, r0, #0
-; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: bfi r1, r3, #0, #1
-; CHECK-NEXT: ubfx r3, r2, #2, #1
+; CHECK-NEXT: vcmp.u32 hi, q0, q1
+; CHECK-NEXT: vmrs r0, p0
+; CHECK-NEXT: and r3, r2, #1
; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #1, #1
+; CHECK-NEXT: bfi r1, r3, #0, #1
; CHECK-NEXT: ubfx r3, r2, #4, #1
; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #2, #1
-; CHECK-NEXT: ubfx r3, r2, #6, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #3, #1
+; CHECK-NEXT: bfi r1, r3, #1, #1
; CHECK-NEXT: ubfx r3, r2, #8, #1
+; CHECK-NEXT: ubfx r2, r2, #12, #1
; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #4, #1
-; CHECK-NEXT: ubfx r3, r2, #10, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #5, #1
-; CHECK-NEXT: ubfx r3, r2, #12, #1
-; CHECK-NEXT: ubfx r2, r2, #14, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r1, r3, #6, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r1, r2, #7, #1
-; CHECK-NEXT: movw r2, :lower16:b
-; CHECK-NEXT: movt r2, :upper16:b
-; CHECK-NEXT: strb r1, [r2]
-; CHECK-NEXT: vmov.u8 r1, q0[0]
-; CHECK-NEXT: vmov.16 q1[0], r1
-; CHECK-NEXT: vmov.u8 r1, q0[1]
-; CHECK-NEXT: vmov.16 q1[1], r1
-; CHECK-NEXT: vmov.u8 r1, q0[2]
-; CHECK-NEXT: vmov.16 q1[2], r1
-; CHECK-NEXT: vmov.u8 r1, q0[3]
-; CHECK-NEXT: vmov.16 q1[3], r1
-; CHECK-NEXT: vmov.u8 r1, q0[4]
-; CHECK-NEXT: vmov.16 q1[4], r1
-; CHECK-NEXT: vmov.u8 r1, q0[5]
-; CHECK-NEXT: vmov.16 q1[5], r1
-; CHECK-NEXT: vmov.u8 r1, q0[6]
-; CHECK-NEXT: vmov.16 q1[6], r1
-; CHECK-NEXT: vmov.u8 r1, q0[7]
-; CHECK-NEXT: vmov.16 q1[7], r1
-; CHECK-NEXT: vcmp.i16 ne, q1, zr
-; CHECK-NEXT: vmrs r1, p0
-; CHECK-NEXT: and r2, r1, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #0, #1
-; CHECK-NEXT: ubfx r2, r1, #2, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #1, #1
-; CHECK-NEXT: ubfx r2, r1, #4, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #2, #1
-; CHECK-NEXT: ubfx r2, r1, #6, #1
+; CHECK-NEXT: bfi r1, r3, #2, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #3, #1
-; CHECK-NEXT: ubfx r2, r1, #8, #1
+; CHECK-NEXT: bfi r1, r2, #3, #1
+; CHECK-NEXT: and r2, r0, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #4, #1
-; CHECK-NEXT: ubfx r2, r1, #10, #1
+; CHECK-NEXT: bfi r1, r2, #4, #1
+; CHECK-NEXT: ubfx r2, r0, #4, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #5, #1
-; CHECK-NEXT: ubfx r2, r1, #12, #1
-; CHECK-NEXT: ubfx r1, r1, #14, #1
+; CHECK-NEXT: bfi r1, r2, #5, #1
+; CHECK-NEXT: ubfx r2, r0, #8, #1
+; CHECK-NEXT: ubfx r0, r0, #12, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #6, #1
-; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: bfi r0, r1, #7, #1
-; CHECK-NEXT: movw r1, :lower16:a
-; CHECK-NEXT: movt r1, :upper16:a
-; CHECK-NEXT: strb r0, [r1]
-; CHECK-NEXT: add sp, #48
-; CHECK-NEXT: bx lr
+; CHECK-NEXT: bfi r1, r2, #6, #1
+; CHECK-NEXT: rsbs r0, r0, #0
+; CHECK-NEXT: bfi r1, r0, #7, #1
+; CHECK-NEXT: movw r0, :lower16:b
+; CHECK-NEXT: movt r0, :upper16:b
+; CHECK-NEXT: strb r1, [r0]
+; CHECK-NEXT: pop {r7, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.1:
; CHECK-NEXT: .LCPI0_0:
-; CHECK-NEXT: .long 12 @ 0xc
-; CHECK-NEXT: .long 13 @ 0xd
-; CHECK-NEXT: .long 14 @ 0xe
-; CHECK-NEXT: .long 15 @ 0xf
+; CHECK-NEXT: .long 0 @ 0x0
+; CHECK-NEXT: .long 1 @ 0x1
+; CHECK-NEXT: .long 2 @ 0x2
+; CHECK-NEXT: .long 3 @ 0x3
; CHECK-NEXT: .LCPI0_1:
-; CHECK-NEXT: .long 8 @ 0x8
-; CHECK-NEXT: .long 9 @ 0x9
-; CHECK-NEXT: .long 10 @ 0xa
-; CHECK-NEXT: .long 11 @ 0xb
-; CHECK-NEXT: .LCPI0_2:
; CHECK-NEXT: .long 4 @ 0x4
; CHECK-NEXT: .long 5 @ 0x5
; CHECK-NEXT: .long 6 @ 0x6
; CHECK-NEXT: .long 7 @ 0x7
-; CHECK-NEXT: .LCPI0_3:
-; CHECK-NEXT: .long 0 @ 0x0
-; CHECK-NEXT: .long 1 @ 0x1
-; CHECK-NEXT: .long 2 @ 0x2
-; CHECK-NEXT: .long 3 @ 0x3
%r = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %i, i32 %n)
%v0 = call <8 x i1> @llvm.vector.extract.v8i1.v16i1.i64(<16 x i1> %r, i64 0)
%v1 = call <8 x i1> @llvm.vector.extract.v8i1.v16i1.i64(<16 x i1> %r, i64 8)
@@ -177,395 +109,403 @@ define void @test_2x16bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #
; CHECK-LABEL: test_2x16bit_mask_with_64bit_index_and_trip_count:
; CHECK: @ %bb.0:
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT: adds.w r12, r0, #17
+; CHECK-NEXT: adds.w r12, r0, #1
+; CHECK-NEXT: vldr s0, .LCPI1_1
+; CHECK-NEXT: vmov q1[2], q1[0], r0, r12
; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r4, r12, r0
-; CHECK-NEXT: sbcs.w r4, lr, r1
-; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r5, r0, #16
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r5, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r5, r12
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r4
+; CHECK-NEXT: subs.w r12, r12, r0
+; CHECK-NEXT: vmov q1[3], q1[1], r1, lr
+; CHECK-NEXT: sbcs.w r12, lr, r1
+; CHECK-NEXT: vmov.f32 s1, s0
+; CHECK-NEXT: csetm r12, lo
+; CHECK-NEXT: vmov s2, r12
+; CHECK-NEXT: vmov.f32 s3, s2
+; CHECK-NEXT: vorr q1, q1, q0
+; CHECK-NEXT: vmov r12, lr, d2
+; CHECK-NEXT: subs.w r12, r12, r2
+; CHECK-NEXT: sbcs.w r12, lr, r3
; CHECK-NEXT: mov.w lr, #0
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r4
-; CHECK-NEXT: mov.w r12, #0
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: mov.w r12, #0
; CHECK-NEXT: bfi lr, r4, #0, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #19
+; CHECK-NEXT: adds.w r9, r0, #3
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #1, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #18
+; CHECK-NEXT: adds r7, r0, #2
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #2, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #21
+; CHECK-NEXT: adds.w r9, r0, #5
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #3, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #20
+; CHECK-NEXT: adds r7, r0, #4
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #4, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #23
+; CHECK-NEXT: adds.w r9, r0, #7
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #5, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #22
+; CHECK-NEXT: adds r7, r0, #6
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #6, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #25
+; CHECK-NEXT: adds.w r9, r0, #9
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #7, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #24
+; CHECK-NEXT: adds.w r7, r0, #8
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #8, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #27
+; CHECK-NEXT: adds.w r9, r0, #11
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #9, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #26
+; CHECK-NEXT: adds.w r7, r0, #10
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #10, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #29
+; CHECK-NEXT: adds.w r9, r0, #13
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #11, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #28
+; CHECK-NEXT: adds.w r7, r0, #12
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r4, r5, d0
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #12, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds.w r9, r0, #31
+; CHECK-NEXT: adds.w r9, r0, #15
; CHECK-NEXT: adc r8, r1, #0
; CHECK-NEXT: subs.w r6, r9, r0
; CHECK-NEXT: sbcs.w r6, r8, r1
; CHECK-NEXT: bfi lr, r4, #13, #1
; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds.w r7, r0, #30
+; CHECK-NEXT: adds.w r7, r0, #14
; CHECK-NEXT: adc r5, r1, #0
; CHECK-NEXT: subs r4, r7, r0
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
+; CHECK-NEXT: vmov q2[2], q2[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vldr s4, .LCPI1_0
-; CHECK-NEXT: vmov r4, r5, d0
-; CHECK-NEXT: vmov.f32 s5, s4
+; CHECK-NEXT: vmov q2[3], q2[1], r5, r8
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r6
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
; CHECK-NEXT: bfi lr, r4, #14, #1
-; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vmov r4, r5, d3
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: adds r7, r0, #1
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r7, r0
-; CHECK-NEXT: vmov q0[3], q0[1], r1, r6
-; CHECK-NEXT: sbcs.w r7, r6, r1
; CHECK-NEXT: bfi lr, r4, #15, #1
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: movw r4, :lower16:b
-; CHECK-NEXT: vmov s6, r7
-; CHECK-NEXT: movt r4, :upper16:b
+; CHECK-NEXT: movw r4, :lower16:a
+; CHECK-NEXT: movt r4, :upper16:a
; CHECK-NEXT: strh.w lr, [r4]
-; CHECK-NEXT: vmov.f32 s7, s6
-; CHECK-NEXT: vorr q0, q0, q1
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #0, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #3
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #1, #1
+; CHECK-NEXT: adds.w lr, r0, #16
+; CHECK-NEXT: adcs r0, r1, #0
+; CHECK-NEXT: adcs r1, r12, #0
+; CHECK-NEXT: itt ne
+; CHECK-NEXT: movne.w r0, #-1
+; CHECK-NEXT: movne.w lr, #-1
+; CHECK-NEXT: adds.w r1, lr, #1
+; CHECK-NEXT: vmov q1[2], q1[0], lr, r1
+; CHECK-NEXT: adc r4, r0, #0
+; CHECK-NEXT: subs.w r1, r1, lr
+; CHECK-NEXT: vmov q1[3], q1[1], r0, r4
+; CHECK-NEXT: sbcs.w r1, r4, r0
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: vmov s2, r1
+; CHECK-NEXT: vmov.f32 s3, s2
+; CHECK-NEXT: vorr q0, q1, q0
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #0, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #1, #1
+; CHECK-NEXT: adds.w r1, lr, #3
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds r4, r0, #2
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #2
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #2, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #5
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #3, #1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #2, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #3, #1
+; CHECK-NEXT: adds.w r1, lr, #5
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds r4, r0, #4
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #4
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #4, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #7
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #5, #1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: b.w .LBB1_2
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI1_1:
+; CHECK-NEXT: .long 0x00000000 @ float 0
+; CHECK-NEXT: .p2align 1
+; CHECK-NEXT: .LBB1_2:
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #4, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #5, #1
+; CHECK-NEXT: adds.w r1, lr, #7
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds r4, r0, #6
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #6
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #6, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #9
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #7, #1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #6, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #7, #1
+; CHECK-NEXT: adds.w r1, lr, #9
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds.w r4, r0, #8
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #8
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #8, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #11
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #9, #1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #8, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #9, #1
+; CHECK-NEXT: adds.w r1, lr, #11
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds.w r4, r0, #10
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #10
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #10, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds.w r8, r0, #13
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs.w r5, r8, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
-; CHECK-NEXT: bfi r12, r7, #11, #1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #10, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #11, #1
+; CHECK-NEXT: adds.w r1, lr, #13
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds.w r4, r0, #12
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r8
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
+; CHECK-NEXT: adds.w r6, lr, #12
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r4, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
+; CHECK-NEXT: vmov q0[2], q0[0], r4, r5
+; CHECK-NEXT: vmov q0[3], q0[1], r4, r5
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #12, #1
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: bfi r12, r7, #13, #1
-; CHECK-NEXT: adds.w r7, r0, #15
-; CHECK-NEXT: adc lr, r1, #0
-; CHECK-NEXT: subs r5, r7, r0
-; CHECK-NEXT: sbcs.w r5, lr, r1
+; CHECK-NEXT: vmov r1, r4, d0
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #12, #1
+; CHECK-NEXT: vmov r1, r4, d1
+; CHECK-NEXT: subs r1, r1, r2
+; CHECK-NEXT: sbcs.w r1, r4, r3
+; CHECK-NEXT: csetm r1, lo
+; CHECK-NEXT: bfi r12, r1, #13, #1
+; CHECK-NEXT: adds.w r1, lr, #15
+; CHECK-NEXT: adc r8, r0, #0
+; CHECK-NEXT: subs.w r5, r1, lr
+; CHECK-NEXT: sbcs.w r5, r8, r0
; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds.w r4, r0, #14
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r0, r4, r0
-; CHECK-NEXT: sbcs.w r0, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, r7
+; CHECK-NEXT: adds.w r6, lr, #14
+; CHECK-NEXT: adc r7, r0, #0
+; CHECK-NEXT: subs.w r4, r6, lr
+; CHECK-NEXT: sbcs.w r0, r7, r0
+; CHECK-NEXT: vmov q1[2], q1[0], r6, r1
; CHECK-NEXT: csetm r0, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, lr
+; CHECK-NEXT: vmov q1[3], q1[1], r7, r8
; CHECK-NEXT: vmov q0[2], q0[0], r0, r5
; CHECK-NEXT: vmov q0[3], q0[1], r0, r5
; CHECK-NEXT: vorr q0, q1, q0
@@ -579,14 +519,11 @@ define void @test_2x16bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #
; CHECK-NEXT: sbcs.w r0, r1, r3
; CHECK-NEXT: csetm r0, lo
; CHECK-NEXT: bfi r12, r0, #15, #1
-; CHECK-NEXT: movw r0, :lower16:a
-; CHECK-NEXT: movt r0, :upper16:a
+; CHECK-NEXT: movw r0, :lower16:b
+; CHECK-NEXT: movt r0, :upper16:b
; CHECK-NEXT: strh.w r12, [r0]
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, pc}
-; CHECK-NEXT: .p2align 2
-; CHECK-NEXT: @ %bb.1:
-; CHECK-NEXT: .LCPI1_0:
-; CHECK-NEXT: .long 0x00000000 @ float 0
+; CHECK-NEXT: @ %bb.3:
%r = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i64 %i, i64 %n)
%v0 = call <16 x i1> @llvm.vector.extract.v16i1.v32i1.i64(<32 x i1> %r, i64 0)
%v1 = call <16 x i1> @llvm.vector.extract.v16i1.v32i1.i64(<32 x i1> %r, i64 16)
@@ -672,146 +609,149 @@ define void @test_2x4bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n
; CHECK-LABEL: test_2x4bit_mask_with_extracts_and_reinterpret_casts:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT: sub sp, #4
-; CHECK-NEXT: adds.w lr, r0, #5
-; CHECK-NEXT: adc r12, r1, #0
-; CHECK-NEXT: subs.w r5, lr, r0
-; CHECK-NEXT: sbcs.w r5, r12, r1
-; CHECK-NEXT: csetm r5, lo
-; CHECK-NEXT: adds r4, r0, #4
-; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r6, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r4, lr
+; CHECK-NEXT: sub sp, #8
+; CHECK-NEXT: adds r5, r0, #4
+; CHECK-NEXT: mov.w r12, #0
+; CHECK-NEXT: adcs r4, r1, #0
+; CHECK-NEXT: vldr s0, .LCPI3_0
+; CHECK-NEXT: adcs r7, r12, #0
+; CHECK-NEXT: itt ne
+; CHECK-NEXT: movne.w r4, #-1
+; CHECK-NEXT: movne.w r5, #-1
+; CHECK-NEXT: adds r7, r5, #1
+; CHECK-NEXT: vmov.f32 s1, s0
+; CHECK-NEXT: vmov q1[2], q1[0], r5, r7
+; CHECK-NEXT: adc r6, r4, #0
+; CHECK-NEXT: subs r7, r7, r5
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r6
+; CHECK-NEXT: sbcs.w r7, r6, r4
; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r6, r12
-; CHECK-NEXT: vmov q0[2], q0[0], r7, r5
-; CHECK-NEXT: vmov q0[3], q0[1], r7, r5
-; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vmov r7, r6, d0
+; CHECK-NEXT: vmov s2, r7
+; CHECK-NEXT: vmov.f32 s3, s2
+; CHECK-NEXT: vorr q1, q1, q0
+; CHECK-NEXT: vmov r7, r6, d2
; CHECK-NEXT: subs r7, r7, r2
; CHECK-NEXT: sbcs.w r7, r6, r3
; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: str r7, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT: vmov r6, r7, d3
+; CHECK-NEXT: subs r6, r6, r2
+; CHECK-NEXT: sbcs.w r6, r7, r3
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds.w r10, r5, #3
; CHECK-NEXT: str r7, [sp] @ 4-byte Spill
-; CHECK-NEXT: vmov r7, r6, d1
-; CHECK-NEXT: subs r7, r7, r2
-; CHECK-NEXT: sbcs.w r7, r6, r3
-; CHECK-NEXT: csetm lr, lo
-; CHECK-NEXT: adds.w r9, r0, #7
-; CHECK-NEXT: adc r8, r1, #0
-; CHECK-NEXT: subs.w r6, r9, r0
-; CHECK-NEXT: sbcs.w r6, r8, r1
-; CHECK-NEXT: csetm r6, lo
-; CHECK-NEXT: adds r7, r0, #6
+; CHECK-NEXT: adc r9, r4, #0
+; CHECK-NEXT: subs.w r7, r10, r5
+; CHECK-NEXT: sbcs.w r7, r9, r4
+; CHECK-NEXT: csetm r7, lo
+; CHECK-NEXT: adds r6, r5, #2
+; CHECK-NEXT: adc r11, r4, #0
+; CHECK-NEXT: subs r5, r6, r5
+; CHECK-NEXT: sbcs.w r4, r11, r4
+; CHECK-NEXT: vmov q2[2], q2[0], r6, r10
+; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: vmov q2[3], q2[1], r11, r9
+; CHECK-NEXT: vmov q1[2], q1[0], r4, r7
+; CHECK-NEXT: vmov q1[3], q1[1], r4, r7
+; CHECK-NEXT: vorr q1, q2, q1
+; CHECK-NEXT: vmov r4, r5, d2
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: vmov r4, r5, d3
+; CHECK-NEXT: csetm r9, lo
+; CHECK-NEXT: subs r4, r4, r2
+; CHECK-NEXT: sbcs.w r4, r5, r3
+; CHECK-NEXT: csetm r10, lo
+; CHECK-NEXT: adds r4, r0, #1
+; CHECK-NEXT: vmov q1[2], q1[0], r0, r4
; CHECK-NEXT: adc r5, r1, #0
-; CHECK-NEXT: subs r4, r7, r0
+; CHECK-NEXT: subs r4, r4, r0
+; CHECK-NEXT: vmov q1[3], q1[1], r1, r5
; CHECK-NEXT: sbcs.w r4, r5, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r7, r9
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r5, r8
-; CHECK-NEXT: vmov q0[2], q0[0], r4, r6
-; CHECK-NEXT: vmov q0[3], q0[1], r4, r6
+; CHECK-NEXT: vmov s2, r4
+; CHECK-NEXT: vmov.f32 s3, s2
; CHECK-NEXT: vorr q0, q1, q0
-; CHECK-NEXT: vldr s4, .LCPI3_0
; CHECK-NEXT: vmov r4, r5, d0
-; CHECK-NEXT: vmov.f32 s5, s4
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
; CHECK-NEXT: vmov r4, r5, d1
-; CHECK-NEXT: csetm r8, lo
+; CHECK-NEXT: cset r7, lo
; CHECK-NEXT: subs r4, r4, r2
; CHECK-NEXT: sbcs.w r4, r5, r3
-; CHECK-NEXT: csetm r9, lo
-; CHECK-NEXT: adds r4, r0, #1
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT: csetm r11, lo
+; CHECK-NEXT: adds r5, r0, #3
; CHECK-NEXT: adc r6, r1, #0
-; CHECK-NEXT: subs r4, r4, r0
-; CHECK-NEXT: vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT: subs r4, r5, r0
; CHECK-NEXT: sbcs.w r4, r6, r1
; CHECK-NEXT: csetm r4, lo
-; CHECK-NEXT: vmov s6, r4
-; CHECK-NEXT: vmov.f32 s7, s6
-; CHECK-NEXT: vorr q0, q0, q1
-; CHECK-NEXT: vmov r4, r6, d0
-; CHECK-NEXT: subs r4, r4, r2
-; CHECK-NEXT: sbcs.w r4, r6, r3
-; CHECK-NEXT: vmov r4, r7, d1
-; CHECK-NEXT: cset r11, lo
-; CHECK-NEXT: subs r4, r4, r2
-; CHECK-NEXT: sbcs.w r4, r7, r3
-; CHECK-NEXT: csetm r10, lo
-; CHECK-NEXT: adds r4, r0, #3
-; CHECK-NEXT: adc r5, r1, #0
-; CHECK-NEXT: subs r7, r4, r0
-; CHECK-NEXT: sbcs.w r7, r5, r1
-; CHECK-NEXT: csetm r7, lo
-; CHECK-NEXT: adds r6, r0, #2
-; CHECK-NEXT: adc r12, r1, #0
-; CHECK-NEXT: subs r0, r6, r0
-; CHECK-NEXT: sbcs.w r0, r12, r1
-; CHECK-NEXT: vmov q1[2], q1[0], r6, r4
+; CHECK-NEXT: adds.w lr, r0, #2
+; CHECK-NEXT: adc r8, r1, #0
+; CHECK-NEXT: subs.w r0, lr, r0
+; CHECK-NEXT: sbcs.w r0, r8, r1
+; CHECK-NEXT: vmov q1[2], q1[0], lr, r5
; CHECK-NEXT: csetm r0, lo
-; CHECK-NEXT: vmov q1[3], q1[1], r12, r5
-; CHECK-NEXT: vmov q0[2], q0[0], r0, r7
-; CHECK-NEXT: vmov q0[3], q0[1], r0, r7
+; CHECK-NEXT: vmov q1[3], q1[1], r8, r6
+; CHECK-NEXT: vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT: vmov q0[3], q0[1], r0, r4
; CHECK-NEXT: vorr q0, q1, q0
; CHECK-NEXT: vmov r0, r1, d0
; CHECK-NEXT: subs r0, r0, r2
; CHECK-NEXT: sbcs.w r0, r1, r3
; CHECK-NEXT: vmov r0, r1, d1
-; CHECK-NEXT: csetm r4, lo
+; CHECK-NEXT: csetm r5, lo
; CHECK-NEXT: subs r0, r0, r2
; CHECK-NEXT: sbcs.w r0, r1, r3
-; CHECK-NEXT: csetm r3, lo
-; CHECK-NEXT: cmp.w r11, #0
+; CHECK-NEXT: csetm r2, lo
+; CHECK-NEXT: cmp r7, #0
; CHECK-NEXT: beq .LBB3_2
; CHECK-NEXT: @ %bb.1: @ %if.then
-; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT: movs r2, #0
-; CHECK-NEXT: rsb.w r7, r11, #0
-; CHECK-NEXT: movs r0, #0
-; CHECK-NEXT: bfi r2, r1, #0, #4
+; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload
; CHECK-NEXT: movs r1, #0
-; CHECK-NEXT: bfi r1, r7, #0, #4
-; CHECK-NEXT: bfi r2, lr, #4, #4
-; CHECK-NEXT: bfi r2, r8, #8, #4
-; CHECK-NEXT: bfi r1, r10, #4, #4
-; CHECK-NEXT: bfi r1, r4, #8, #4
-; CHECK-NEXT: bfi r2, r9, #12, #4
-; CHECK-NEXT: bfi r1, r3, #12, #4
-; CHECK-NEXT: and r3, r2, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: movs r7, #0
-; CHECK-NEXT: bfi r7, r3, #0, #1
-; CHECK-NEXT: ubfx r3, r2, #4, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r7, r3, #1, #1
-; CHECK-NEXT: ubfx r3, r2, #8, #1
-; CHECK-NEXT: ubfx r2, r2, #12, #1
-; CHECK-NEXT: rsbs r3, r3, #0
-; CHECK-NEXT: bfi r7, r3, #2, #1
-; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r7, r2, #3, #1
-; CHECK-NEXT: movw r2, :lower16:b
-; CHECK-NEXT: movt r2, :upper16:b
-; CHECK-NEXT: strb r7, [r2]
+; CHECK-NEXT: rsbs r3, r7, #0
+; CHECK-NEXT: bfi r1, r0, #0, #4
+; CHECK-NEXT: ldr r0, [sp] @ 4-byte Reload
+; CHECK-NEXT: bfi r1, r0, #4, #4
+; CHECK-NEXT: movs r0, #0
+; CHECK-NEXT: bfi r0, r3, #0, #4
+; CHECK-NEXT: bfi r1, r9, #8, #4
+; CHECK-NEXT: bfi r0, r11, #4, #4
+; CHECK-NEXT: bfi r1, r10, #12, #4
+; CHECK-NEXT: bfi r0, r5, #8, #4
+; CHECK-NEXT: movs r3, #0
+; CHECK-NEXT: bfi r0, r2, #12, #4
; CHECK-NEXT: and r2, r1, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #0, #1
+; CHECK-NEXT: bfi r3, r2, #0, #1
; CHECK-NEXT: ubfx r2, r1, #4, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #1, #1
+; CHECK-NEXT: bfi r3, r2, #1, #1
; CHECK-NEXT: ubfx r2, r1, #8, #1
; CHECK-NEXT: ubfx r1, r1, #12, #1
; CHECK-NEXT: rsbs r2, r2, #0
-; CHECK-NEXT: bfi r0, r2, #2, #1
+; CHECK-NEXT: bfi r3, r2, #2, #1
+; CHECK-NEXT: rsbs r1, r1, #0
+; CHECK-NEXT: bfi r3, r1, #3, #1
+; CHECK-NEXT: movw r1, :lower16:b
+; CHECK-NEXT: movt r1, :upper16:b
+; CHECK-NEXT: strb r3, [r1]
+; CHECK-NEXT: and r1, r0, #1
+; CHECK-NEXT: rsbs r1, r1, #0
+; CHECK-NEXT: bfi r12, r1, #0, #1
+; CHECK-NEXT: ubfx r1, r0, #4, #1
+; CHECK-NEXT: rsbs r1, r1, #0
+; CHECK-NEXT: bfi r12, r1, #1, #1
+; CHECK-NEXT: ubfx r1, r0, #8, #1
+; CHECK-NEXT: ubfx r0, r0, #12, #1
; CHECK-NEXT: rsbs r1, r1, #0
-; CHECK-NEXT: bfi r0, r1, #3, #1
-; CHECK-NEXT: movw r1, :lower16:a
-; CHECK-NEXT: movt r1, :upper16:a
-; CHECK-NEXT: strb r0, [r1]
+; CHECK-NEXT: bfi r12, r1, #2, #1
+; CHECK-NEXT: rsbs r0, r0, #0
+; CHECK-NEXT: bfi r12, r0, #3, #1
+; CHECK-NEXT: movw r0, :lower16:a
+; CHECK-NEXT: movt r0, :upper16:a
+; CHECK-NEXT: strb.w r12, [r0]
; CHECK-NEXT: .LBB3_2: @ %if.end
-; CHECK-NEXT: add sp, #4
+; CHECK-NEXT: add sp, #8
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .p2align 2
; CHECK-NEXT: @ %bb.3:
More information about the llvm-commits
mailing list