[llvm] [AArch64] Lower bitcast of wide <N x i1> masks without a stack spill in more cases (PR #203420)
Joe Isaacs via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 03:38:39 PDT 2026
https://github.com/joseph-isaacs updated https://github.com/llvm/llvm-project/pull/203420
>From 84db1b5d887d70f2fe53df8b10d2cf617409e91c Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs at live.co.uk>
Date: Wed, 10 Jun 2026 16:02:46 +0100
Subject: [PATCH 1/2] [AArch64] Narrow lane type in vectorToScalarBitmask
instead of bailing on >128-bit source vector type
vectorToScalarBitmask() lowers `bitcast <N x i1> to iN` to a register-only
movemask, but bailed when the mask's source vector type `<N x ty>` is wider
than 128 bits, leaving the bitcast to legalize through a stack temporary.
The bail is unnecessary: the bitmask depends only on the lane count and
order, never on the per-lane width (sign-extending to the source width was
only ever a free-sext optimization). Narrow the per-lane integer width to
128/NumElts bits so the mask fits one 128-bit register and the existing
register-only lowering applies.
Fixes #203414
Co-Authored-By: Claude Fable 5 <noreply at anthropic.com>
---
.../Target/AArch64/AArch64ISelLowering.cpp | 15 +-
.../sve-fixed-length-masked-expandloads.ll | 1935 +++++------------
.../vec-combine-compare-to-bitmask-wide.ll | 339 +++
.../AArch64/vec-combine-compare-to-bitmask.ll | 6 +-
4 files changed, 867 insertions(+), 1428 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 57a2d73e00f57..f585ba4bdefb4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26193,11 +26193,16 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
}
VecVT = VecVT.changeVectorElementTypeToInteger();
- // Large vectors don't map directly to this conversion, so to avoid too many
- // edge cases, we don't apply it here. The conversion will likely still be
- // applied later via multiple smaller vectors, whose results are concatenated.
- if (VecVT.getSizeInBits() > 128)
- return SDValue();
+ // The bitmask depends only on the lane count, not the lane width so when
+ // the preferred type exceeds 128 bits narrow the lanes to fill one 128-bit
+ // register instead of spilling through the stack.
+ if (VecVT.getSizeInBits() > 128) {
+ assert(128 % NumElts == 0 && "Expected NumElts to evenly divide 128 bits");
+ unsigned NarrowedEltBits = 128 / NumElts;
+ if (NarrowedEltBits < 8)
+ return SDValue();
+ VecVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowedEltBits), NumElts);
+ }
// Ensure that all elements' bits are either 0s or 1s.
ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
index f2ddac3e9fac7..f7abcc3f63d9a 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-expandloads.ll
@@ -192,30 +192,17 @@ define void @masked_load_v8f32(ptr %ap, ptr %bp, ptr %c) vscale_range(2,0) #0 {
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: ptrue p0.s, vl8
+; CHECK-NEXT: adrp x8, .LCPI3_0
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT: umov w8, v0.b[0]
-; CHECK-NEXT: umov w9, v0.b[1]
-; CHECK-NEXT: umov w10, v0.b[2]
-; CHECK-NEXT: and w8, w8, #0x1
-; CHECK-NEXT: bfi w8, w9, #1, #1
-; CHECK-NEXT: umov w9, v0.b[3]
-; CHECK-NEXT: bfi w8, w10, #2, #1
-; CHECK-NEXT: umov w10, v0.b[4]
-; CHECK-NEXT: bfi w8, w9, #3, #1
-; CHECK-NEXT: umov w9, v0.b[5]
-; CHECK-NEXT: bfi w8, w10, #4, #1
-; CHECK-NEXT: umov w10, v0.b[6]
-; CHECK-NEXT: bfi w8, w9, #5, #1
-; CHECK-NEXT: umov w9, v0.b[7]
-; CHECK-NEXT: bfi w8, w10, #6, #1
-; CHECK-NEXT: orr w9, w8, w9, lsl #7
-; CHECK-NEXT: and w8, w9, #0xff
-; CHECK-NEXT: tbz w9, #0, .LBB3_2
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tbz w8, #0, .LBB3_2
; CHECK-NEXT: // %bb.1: // %cond.load
; CHECK-NEXT: ldr s0, [x0], #4
; CHECK-NEXT: stp xzr, xzr, [sp, #16]
@@ -360,9 +347,8 @@ define void @masked_load_v16f32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB4_2
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB4_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldr s0, [x0], #4
; VBITS_GE_256-NEXT: stp xzr, xzr, [sp, #16]
@@ -554,54 +540,20 @@ define void @masked_load_v16f32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: .cfi_offset w30, -8
; VBITS_GE_512-NEXT: .cfi_offset w29, -16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI4_0
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB4_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB4_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldr s0, [x0], #4
; VBITS_GE_512-NEXT: stp xzr, xzr, [sp, #48]
@@ -787,51 +739,36 @@ define void @masked_load_v16f32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; CHECK-EXPAND-LABEL: masked_load_v16f32:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.s, vl8
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI4_0
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-EXPAND-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s
+; CHECK-EXPAND-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-EXPAND-NEXT: mov x8, #8 // =0x8
+; CHECK-EXPAND-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-EXPAND-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2]
; CHECK-EXPAND-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-EXPAND-NEXT: cntp x11, p1, p1.s
+; CHECK-EXPAND-NEXT: fcmeq p2.s, p0/z, z2.s, z3.s
; CHECK-EXPAND-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-EXPAND-NEXT: uzp1 z0.b, z0.b, z0.b
-; CHECK-EXPAND-NEXT: umov w8, v0.b[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.b[1]
-; CHECK-EXPAND-NEXT: umov w10, v0.b[2]
-; CHECK-EXPAND-NEXT: umov w11, v0.b[3]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.b[4]
-; CHECK-EXPAND-NEXT: bfi w8, w10, #2, #1
-; CHECK-EXPAND-NEXT: umov w10, v0.b[5]
-; CHECK-EXPAND-NEXT: bfi w8, w11, #3, #1
-; CHECK-EXPAND-NEXT: mov x11, #8 // =0x8
-; CHECK-EXPAND-NEXT: ld1w { z1.s }, p0/z, [x1, x11, lsl #2]
-; CHECK-EXPAND-NEXT: ld1w { z2.s }, p0/z, [x0, x11, lsl #2]
-; CHECK-EXPAND-NEXT: bfi w8, w9, #4, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.b[6]
-; CHECK-EXPAND-NEXT: bfi w8, w10, #5, #1
-; CHECK-EXPAND-NEXT: umov w10, v0.b[7]
-; CHECK-EXPAND-NEXT: fcmeq p2.s, p0/z, z2.s, z1.s
-; CHECK-EXPAND-NEXT: bfi w8, w9, #6, #1
-; CHECK-EXPAND-NEXT: orr w8, w8, w10, lsl #7
-; CHECK-EXPAND-NEXT: cntp x10, p1, p1.s
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.s
-; CHECK-EXPAND-NEXT: and w8, w8, #0xff
-; CHECK-EXPAND-NEXT: fmov s0, w8
-; CHECK-EXPAND-NEXT: whilelo p4.s, xzr, x10
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.s, xzr, x9
+; CHECK-EXPAND-NEXT: whilelo p4.s, xzr, x11
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.s
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-EXPAND-NEXT: ld1w { z1.s }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: addv h0, v0.8h
; CHECK-EXPAND-NEXT: expand z1.s, p1, z1.s
-; CHECK-EXPAND-NEXT: ld1w { z0.s }, p3/z, [x0, x8, lsl #2]
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: and w9, w9, #0xff
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.s, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1w { z0.s }, p3/z, [x0, x9, lsl #2]
; CHECK-EXPAND-NEXT: st1w { z1.s }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.s, p2, z0.s
-; CHECK-EXPAND-NEXT: st1w { z0.s }, p0, [x2, x11, lsl #2]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1w { z0.s }, p0, [x2, x8, lsl #2]
; CHECK-EXPAND-NEXT: ret
%a = load <16 x float>, ptr %ap
%b = load <16 x float>, ptr %bp
@@ -5254,74 +5191,37 @@ define void @masked_load_v32i16(ptr %ap, ptr %bp, ptr %c) #0 {
;
; CHECK-EXPAND-LABEL: masked_load_v32i16:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.h, vl16
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI8_0
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1h { z0.h }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1h { z1.h }, p0/z, [x1]
; CHECK-EXPAND-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-EXPAND-NEXT: ldr q1, [x8, :lo12:.LCPI8_0]
+; CHECK-EXPAND-NEXT: mov x8, #16 // =0x10
+; CHECK-EXPAND-NEXT: ld1h { z2.h }, p0/z, [x1, x8, lsl #1]
; CHECK-EXPAND-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-EXPAND-NEXT: cntp x11, p1, p1.h
; CHECK-EXPAND-NEXT: uzp1 z0.b, z0.b, z0.b
-; CHECK-EXPAND-NEXT: umov w8, v0.b[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.b[1]
-; CHECK-EXPAND-NEXT: umov w10, v0.b[7]
-; CHECK-EXPAND-NEXT: umov w11, v0.b[8]
-; CHECK-EXPAND-NEXT: umov w12, v0.b[2]
-; CHECK-EXPAND-NEXT: umov w13, v0.b[9]
-; CHECK-EXPAND-NEXT: umov w14, v0.b[10]
-; CHECK-EXPAND-NEXT: umov w15, v0.b[3]
-; CHECK-EXPAND-NEXT: umov w16, v0.b[4]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: ubfiz w10, w10, #7, #1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.b[11]
-; CHECK-EXPAND-NEXT: ubfiz w11, w11, #8, #1
-; CHECK-EXPAND-NEXT: ubfiz w13, w13, #9, #1
-; CHECK-EXPAND-NEXT: ubfiz w14, w14, #10, #1
-; CHECK-EXPAND-NEXT: bfi w8, w12, #2, #1
-; CHECK-EXPAND-NEXT: orr w10, w10, w11
-; CHECK-EXPAND-NEXT: umov w11, v0.b[12]
-; CHECK-EXPAND-NEXT: umov w12, v0.b[5]
-; CHECK-EXPAND-NEXT: orr w10, w10, w13
-; CHECK-EXPAND-NEXT: umov w13, v0.b[13]
-; CHECK-EXPAND-NEXT: bfi w8, w15, #3, #1
-; CHECK-EXPAND-NEXT: umov w15, v0.b[14]
-; CHECK-EXPAND-NEXT: ubfiz w9, w9, #11, #1
-; CHECK-EXPAND-NEXT: orr w10, w10, w14
-; CHECK-EXPAND-NEXT: mov x14, #16 // =0x10
-; CHECK-EXPAND-NEXT: bfi w8, w16, #4, #1
-; CHECK-EXPAND-NEXT: umov w16, v0.b[6]
-; CHECK-EXPAND-NEXT: orr w9, w10, w9
-; CHECK-EXPAND-NEXT: ubfiz w10, w11, #12, #1
-; CHECK-EXPAND-NEXT: ubfiz w11, w13, #13, #1
-; CHECK-EXPAND-NEXT: ld1h { z1.h }, p0/z, [x1, x14, lsl #1]
-; CHECK-EXPAND-NEXT: bfi w8, w12, #5, #1
-; CHECK-EXPAND-NEXT: ubfiz w12, w15, #14, #1
-; CHECK-EXPAND-NEXT: ld1h { z2.h }, p0/z, [x0, x14, lsl #1]
-; CHECK-EXPAND-NEXT: orr w9, w9, w10
-; CHECK-EXPAND-NEXT: umov w10, v0.b[15]
-; CHECK-EXPAND-NEXT: orr w9, w9, w11
-; CHECK-EXPAND-NEXT: bfi w8, w16, #6, #1
-; CHECK-EXPAND-NEXT: orr w9, w9, w12
-; CHECK-EXPAND-NEXT: cmpeq p2.h, p0/z, z2.h, z1.h
-; CHECK-EXPAND-NEXT: orr w8, w8, w9
-; CHECK-EXPAND-NEXT: orr w8, w8, w10, lsl #15
-; CHECK-EXPAND-NEXT: cntp x10, p1, p1.h
-; CHECK-EXPAND-NEXT: and w8, w8, #0xffff
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.h
-; CHECK-EXPAND-NEXT: fmov s0, w8
-; CHECK-EXPAND-NEXT: whilelo p4.h, xzr, x10
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.h, xzr, x9
+; CHECK-EXPAND-NEXT: whilelo p4.h, xzr, x11
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-EXPAND-NEXT: ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
+; CHECK-EXPAND-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-EXPAND-NEXT: cmpeq p2.h, p0/z, z1.h, z2.h
; CHECK-EXPAND-NEXT: ld1h { z1.h }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: addp v0.16b, v0.16b, v0.16b
; CHECK-EXPAND-NEXT: expand z1.h, p1, z1.h
-; CHECK-EXPAND-NEXT: ld1h { z0.h }, p3/z, [x0, x8, lsl #1]
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.h
+; CHECK-EXPAND-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-EXPAND-NEXT: umov w9, v0.h[0]
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.h, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1h { z0.h }, p3/z, [x0, x9, lsl #1]
; CHECK-EXPAND-NEXT: st1h { z1.h }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.h, p2, z0.h
-; CHECK-EXPAND-NEXT: st1h { z0.h }, p0, [x2, x14, lsl #1]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1h { z0.h }, p0, [x2, x8, lsl #1]
; CHECK-EXPAND-NEXT: ret
%a = load <32 x i16>, ptr %ap
%b = load <32 x i16>, ptr %bp
@@ -5334,12 +5234,10 @@ define void @masked_load_v32i16(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_v16i32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
-; VBITS_GE_256-NEXT: adrp x10, .LCPI9_1
-; VBITS_GE_256-NEXT: add x10, x10, :lo12:.LCPI9_1
+; VBITS_GE_256-NEXT: adrp x9, .LCPI9_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI9_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1, x8, lsl #2]
@@ -5360,10 +5258,9 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x10]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB9_2
+; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x9]
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB9_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z2.s }, p1/z, [x0]
; VBITS_GE_256-NEXT: mov z1.d, z0.d
@@ -5420,7 +5317,6 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB9_20: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -5527,66 +5423,30 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_v16i32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI9_0
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI9_0]
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB9_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB9_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB9_3
; VBITS_GE_512-NEXT: b .LBB9_4
; VBITS_GE_512-NEXT: .LBB9_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI9_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI9_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI9_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI9_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB9_4
; VBITS_GE_512-NEXT: .LBB9_3: // %cond.load1
@@ -5597,45 +5457,37 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: .LBB9_4: // %else2
-; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB9_20
+; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB9_19
; VBITS_GE_512-NEXT: // %bb.5: // %else6
-; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB9_21
+; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB9_20
; VBITS_GE_512-NEXT: .LBB9_6: // %else10
-; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB9_22
+; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB9_21
; VBITS_GE_512-NEXT: .LBB9_7: // %else14
-; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB9_23
+; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB9_22
; VBITS_GE_512-NEXT: .LBB9_8: // %else18
-; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB9_24
+; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB9_23
; VBITS_GE_512-NEXT: .LBB9_9: // %else22
-; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB9_25
+; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB9_24
; VBITS_GE_512-NEXT: .LBB9_10: // %else26
-; VBITS_GE_512-NEXT: tbnz w8, #8, .LBB9_26
+; VBITS_GE_512-NEXT: tbnz w8, #8, .LBB9_25
; VBITS_GE_512-NEXT: .LBB9_11: // %else30
-; VBITS_GE_512-NEXT: tbnz w8, #9, .LBB9_27
+; VBITS_GE_512-NEXT: tbnz w8, #9, .LBB9_26
; VBITS_GE_512-NEXT: .LBB9_12: // %else34
-; VBITS_GE_512-NEXT: tbnz w8, #10, .LBB9_28
+; VBITS_GE_512-NEXT: tbnz w8, #10, .LBB9_27
; VBITS_GE_512-NEXT: .LBB9_13: // %else38
-; VBITS_GE_512-NEXT: tbnz w8, #11, .LBB9_29
+; VBITS_GE_512-NEXT: tbnz w8, #11, .LBB9_28
; VBITS_GE_512-NEXT: .LBB9_14: // %else42
-; VBITS_GE_512-NEXT: tbnz w8, #12, .LBB9_30
+; VBITS_GE_512-NEXT: tbnz w8, #12, .LBB9_29
; VBITS_GE_512-NEXT: .LBB9_15: // %else46
-; VBITS_GE_512-NEXT: tbnz w8, #13, .LBB9_31
+; VBITS_GE_512-NEXT: tbnz w8, #13, .LBB9_30
; VBITS_GE_512-NEXT: .LBB9_16: // %else50
-; VBITS_GE_512-NEXT: tbnz w8, #14, .LBB9_32
+; VBITS_GE_512-NEXT: tbnz w8, #14, .LBB9_31
; VBITS_GE_512-NEXT: .LBB9_17: // %else54
-; VBITS_GE_512-NEXT: tbz w8, #15, .LBB9_19
-; VBITS_GE_512-NEXT: .LBB9_18: // %cond.load57
-; VBITS_GE_512-NEXT: mov w8, #15 // =0xf
-; VBITS_GE_512-NEXT: index z1.s, #0, #1
-; VBITS_GE_512-NEXT: mov z2.s, w8
-; VBITS_GE_512-NEXT: ldr w8, [x0]
-; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
-; VBITS_GE_512-NEXT: mov z0.s, p2/m, w8
-; VBITS_GE_512-NEXT: .LBB9_19: // %else58
+; VBITS_GE_512-NEXT: tbnz w8, #15, .LBB9_32
+; VBITS_GE_512-NEXT: .LBB9_18: // %else58
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
-; VBITS_GE_512-NEXT: .LBB9_20: // %cond.load5
+; VBITS_GE_512-NEXT: .LBB9_19: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5643,7 +5495,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #3, .LBB9_6
-; VBITS_GE_512-NEXT: .LBB9_21: // %cond.load9
+; VBITS_GE_512-NEXT: .LBB9_20: // %cond.load9
; VBITS_GE_512-NEXT: mov w9, #3 // =0x3
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5651,7 +5503,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #4, .LBB9_7
-; VBITS_GE_512-NEXT: .LBB9_22: // %cond.load13
+; VBITS_GE_512-NEXT: .LBB9_21: // %cond.load13
; VBITS_GE_512-NEXT: mov w9, #4 // =0x4
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5659,7 +5511,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #5, .LBB9_8
-; VBITS_GE_512-NEXT: .LBB9_23: // %cond.load17
+; VBITS_GE_512-NEXT: .LBB9_22: // %cond.load17
; VBITS_GE_512-NEXT: mov w9, #5 // =0x5
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5667,7 +5519,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #6, .LBB9_9
-; VBITS_GE_512-NEXT: .LBB9_24: // %cond.load21
+; VBITS_GE_512-NEXT: .LBB9_23: // %cond.load21
; VBITS_GE_512-NEXT: mov w9, #6 // =0x6
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5675,7 +5527,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #7, .LBB9_10
-; VBITS_GE_512-NEXT: .LBB9_25: // %cond.load25
+; VBITS_GE_512-NEXT: .LBB9_24: // %cond.load25
; VBITS_GE_512-NEXT: mov w9, #7 // =0x7
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5683,7 +5535,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #8, .LBB9_11
-; VBITS_GE_512-NEXT: .LBB9_26: // %cond.load29
+; VBITS_GE_512-NEXT: .LBB9_25: // %cond.load29
; VBITS_GE_512-NEXT: mov w9, #8 // =0x8
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5691,7 +5543,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #9, .LBB9_12
-; VBITS_GE_512-NEXT: .LBB9_27: // %cond.load33
+; VBITS_GE_512-NEXT: .LBB9_26: // %cond.load33
; VBITS_GE_512-NEXT: mov w9, #9 // =0x9
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5699,7 +5551,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #10, .LBB9_13
-; VBITS_GE_512-NEXT: .LBB9_28: // %cond.load37
+; VBITS_GE_512-NEXT: .LBB9_27: // %cond.load37
; VBITS_GE_512-NEXT: mov w9, #10 // =0xa
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5707,7 +5559,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #11, .LBB9_14
-; VBITS_GE_512-NEXT: .LBB9_29: // %cond.load41
+; VBITS_GE_512-NEXT: .LBB9_28: // %cond.load41
; VBITS_GE_512-NEXT: mov w9, #11 // =0xb
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5715,7 +5567,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #12, .LBB9_15
-; VBITS_GE_512-NEXT: .LBB9_30: // %cond.load45
+; VBITS_GE_512-NEXT: .LBB9_29: // %cond.load45
; VBITS_GE_512-NEXT: mov w9, #12 // =0xc
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5723,7 +5575,7 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #13, .LBB9_16
-; VBITS_GE_512-NEXT: .LBB9_31: // %cond.load49
+; VBITS_GE_512-NEXT: .LBB9_30: // %cond.load49
; VBITS_GE_512-NEXT: mov w9, #13 // =0xd
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
@@ -5731,63 +5583,56 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
; VBITS_GE_512-NEXT: tbz w8, #14, .LBB9_17
-; VBITS_GE_512-NEXT: .LBB9_32: // %cond.load53
+; VBITS_GE_512-NEXT: .LBB9_31: // %cond.load53
; VBITS_GE_512-NEXT: mov w9, #14 // =0xe
; VBITS_GE_512-NEXT: index z1.s, #0, #1
; VBITS_GE_512-NEXT: mov z2.s, w9
; VBITS_GE_512-NEXT: ldr w9, [x0], #4
; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
; VBITS_GE_512-NEXT: mov z0.s, p2/m, w9
-; VBITS_GE_512-NEXT: tbnz w8, #15, .LBB9_18
-; VBITS_GE_512-NEXT: b .LBB9_19
+; VBITS_GE_512-NEXT: tbz w8, #15, .LBB9_18
+; VBITS_GE_512-NEXT: .LBB9_32: // %cond.load57
+; VBITS_GE_512-NEXT: mov w8, #15 // =0xf
+; VBITS_GE_512-NEXT: index z1.s, #0, #1
+; VBITS_GE_512-NEXT: mov z2.s, w8
+; VBITS_GE_512-NEXT: ldr w8, [x0]
+; VBITS_GE_512-NEXT: cmpeq p2.s, p1/z, z1.s, z2.s
+; VBITS_GE_512-NEXT: mov z0.s, p2/m, w8
+; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
+; VBITS_GE_512-NEXT: ret
;
; CHECK-EXPAND-LABEL: masked_load_v16i32:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.s, vl8
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI9_0
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1w { z0.s }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1w { z1.s }, p0/z, [x1]
; CHECK-EXPAND-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-EXPAND-NEXT: ldr q1, [x8, :lo12:.LCPI9_0]
+; CHECK-EXPAND-NEXT: mov x8, #8 // =0x8
+; CHECK-EXPAND-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-EXPAND-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2]
; CHECK-EXPAND-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-EXPAND-NEXT: cmpeq p2.s, p0/z, z2.s, z3.s
+; CHECK-EXPAND-NEXT: cntp x11, p1, p1.s
; CHECK-EXPAND-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-EXPAND-NEXT: uzp1 z0.b, z0.b, z0.b
-; CHECK-EXPAND-NEXT: umov w8, v0.b[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.b[1]
-; CHECK-EXPAND-NEXT: umov w10, v0.b[2]
-; CHECK-EXPAND-NEXT: umov w11, v0.b[3]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.b[4]
-; CHECK-EXPAND-NEXT: bfi w8, w10, #2, #1
-; CHECK-EXPAND-NEXT: umov w10, v0.b[5]
-; CHECK-EXPAND-NEXT: bfi w8, w11, #3, #1
-; CHECK-EXPAND-NEXT: mov x11, #8 // =0x8
-; CHECK-EXPAND-NEXT: ld1w { z1.s }, p0/z, [x1, x11, lsl #2]
-; CHECK-EXPAND-NEXT: ld1w { z2.s }, p0/z, [x0, x11, lsl #2]
-; CHECK-EXPAND-NEXT: bfi w8, w9, #4, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.b[6]
-; CHECK-EXPAND-NEXT: bfi w8, w10, #5, #1
-; CHECK-EXPAND-NEXT: umov w10, v0.b[7]
-; CHECK-EXPAND-NEXT: cmpeq p2.s, p0/z, z2.s, z1.s
-; CHECK-EXPAND-NEXT: bfi w8, w9, #6, #1
-; CHECK-EXPAND-NEXT: orr w8, w8, w10, lsl #7
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.s
-; CHECK-EXPAND-NEXT: cntp x10, p1, p1.s
-; CHECK-EXPAND-NEXT: and w8, w8, #0xff
-; CHECK-EXPAND-NEXT: fmov s0, w8
-; CHECK-EXPAND-NEXT: whilelo p4.s, xzr, x10
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.s, xzr, x9
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.s
+; CHECK-EXPAND-NEXT: whilelo p4.s, xzr, x11
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-EXPAND-NEXT: ld1w { z1.s }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: ld1w { z0.s }, p3/z, [x0, x8, lsl #2]
+; CHECK-EXPAND-NEXT: addv h0, v0.8h
; CHECK-EXPAND-NEXT: expand z1.s, p1, z1.s
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: and w9, w9, #0xff
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.s, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1w { z0.s }, p3/z, [x0, x9, lsl #2]
; CHECK-EXPAND-NEXT: st1w { z1.s }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.s, p2, z0.s
-; CHECK-EXPAND-NEXT: st1w { z0.s }, p0, [x2, x11, lsl #2]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1w { z0.s }, p0, [x2, x8, lsl #2]
; CHECK-EXPAND-NEXT: ret
%a = load <16 x i32>, ptr %ap
%b = load <16 x i32>, ptr %bp
@@ -5800,14 +5645,13 @@ define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_v8i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI10_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, z2.d
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, z3.d
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -5818,28 +5662,14 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: ptrue p1.d
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: adrp x8, .LCPI10_0
-; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI10_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI10_1
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI10_1
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h1, v0.8h
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x8]
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB10_2
+; VBITS_GE_256-NEXT: fmov w8, s1
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB10_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rd { z2.d }, p1/z, [x0]
; VBITS_GE_256-NEXT: mov z1.d, z0.d
@@ -5880,7 +5710,6 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB10_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -5923,43 +5752,28 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_v8i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI10_0
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.d
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB10_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB10_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rd { z0.d }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #8
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB10_3
; VBITS_GE_512-NEXT: b .LBB10_4
; VBITS_GE_512-NEXT: .LBB10_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI10_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI10_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI10_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI10_1
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB10_4
; VBITS_GE_512-NEXT: .LBB10_3: // %cond.load1
@@ -5970,29 +5784,21 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: .LBB10_4: // %else2
-; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB10_12
+; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB10_11
; VBITS_GE_512-NEXT: // %bb.5: // %else6
-; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB10_13
+; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB10_12
; VBITS_GE_512-NEXT: .LBB10_6: // %else10
-; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB10_14
+; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB10_13
; VBITS_GE_512-NEXT: .LBB10_7: // %else14
-; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB10_15
+; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB10_14
; VBITS_GE_512-NEXT: .LBB10_8: // %else18
-; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB10_16
+; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB10_15
; VBITS_GE_512-NEXT: .LBB10_9: // %else22
-; VBITS_GE_512-NEXT: tbz w8, #7, .LBB10_11
-; VBITS_GE_512-NEXT: .LBB10_10: // %cond.load25
-; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
-; VBITS_GE_512-NEXT: index z1.d, #0, #1
-; VBITS_GE_512-NEXT: mov z2.d, x8
-; VBITS_GE_512-NEXT: ldr x8, [x0]
-; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
-; VBITS_GE_512-NEXT: mov z0.d, p2/m, x8
-; VBITS_GE_512-NEXT: .LBB10_11: // %else26
+; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB10_16
+; VBITS_GE_512-NEXT: .LBB10_10: // %else26
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
-; VBITS_GE_512-NEXT: .LBB10_12: // %cond.load5
+; VBITS_GE_512-NEXT: .LBB10_11: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: mov z2.d, x9
@@ -6000,7 +5806,7 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #3, .LBB10_6
-; VBITS_GE_512-NEXT: .LBB10_13: // %cond.load9
+; VBITS_GE_512-NEXT: .LBB10_12: // %cond.load9
; VBITS_GE_512-NEXT: mov w9, #3 // =0x3
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: mov z2.d, x9
@@ -6008,7 +5814,7 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #4, .LBB10_7
-; VBITS_GE_512-NEXT: .LBB10_14: // %cond.load13
+; VBITS_GE_512-NEXT: .LBB10_13: // %cond.load13
; VBITS_GE_512-NEXT: mov w9, #4 // =0x4
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: mov z2.d, x9
@@ -6016,7 +5822,7 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #5, .LBB10_8
-; VBITS_GE_512-NEXT: .LBB10_15: // %cond.load17
+; VBITS_GE_512-NEXT: .LBB10_14: // %cond.load17
; VBITS_GE_512-NEXT: mov w9, #5 // =0x5
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: mov z2.d, x9
@@ -6024,55 +5830,56 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #6, .LBB10_9
-; VBITS_GE_512-NEXT: .LBB10_16: // %cond.load21
+; VBITS_GE_512-NEXT: .LBB10_15: // %cond.load21
; VBITS_GE_512-NEXT: mov w9, #6 // =0x6
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: mov z2.d, x9
; VBITS_GE_512-NEXT: ldr x9, [x0], #8
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
-; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB10_10
-; VBITS_GE_512-NEXT: b .LBB10_11
+; VBITS_GE_512-NEXT: tbz w8, #7, .LBB10_10
+; VBITS_GE_512-NEXT: .LBB10_16: // %cond.load25
+; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
+; VBITS_GE_512-NEXT: index z1.d, #0, #1
+; VBITS_GE_512-NEXT: mov z2.d, x8
+; VBITS_GE_512-NEXT: ldr x8, [x0]
+; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
+; VBITS_GE_512-NEXT: mov z0.d, p2/m, x8
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
+; VBITS_GE_512-NEXT: ret
;
; CHECK-EXPAND-LABEL: masked_load_v8i64:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.d, vl4
-; CHECK-EXPAND-NEXT: mov x10, #4 // =0x4
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI10_0
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1d { z1.d }, p0/z, [x1]
-; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x0, x10, lsl #3]
; CHECK-EXPAND-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
-; CHECK-EXPAND-NEXT: ld1d { z1.d }, p0/z, [x1, x10, lsl #3]
-; CHECK-EXPAND-NEXT: cmpeq p2.d, p0/z, z2.d, z1.d
+; CHECK-EXPAND-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
+; CHECK-EXPAND-NEXT: mov x8, #4 // =0x4
+; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
+; CHECK-EXPAND-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3]
; CHECK-EXPAND-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
-; CHECK-EXPAND-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-EXPAND-NEXT: umov w8, v0.h[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.h[1]
-; CHECK-EXPAND-NEXT: umov w11, v0.h[2]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.h[3]
-; CHECK-EXPAND-NEXT: bfi w8, w11, #2, #1
+; CHECK-EXPAND-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d
; CHECK-EXPAND-NEXT: cntp x11, p1, p1.d
-; CHECK-EXPAND-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.d
-; CHECK-EXPAND-NEXT: and w8, w8, #0xf
+; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.d
; CHECK-EXPAND-NEXT: whilelo p4.d, xzr, x11
-; CHECK-EXPAND-NEXT: fmov s0, w8
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-EXPAND-NEXT: ld1d { z1.d }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x9
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: addv s0, v0.4s
; CHECK-EXPAND-NEXT: expand z1.d, p1, z1.d
-; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x8, lsl #3]
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: and w9, w9, #0xf
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x9, lsl #3]
; CHECK-EXPAND-NEXT: st1d { z1.d }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.d, p2, z0.d
-; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x10, lsl #3]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]
; CHECK-EXPAND-NEXT: ret
%a = load <8 x i64>, ptr %ap
%b = load <8 x i64>, ptr %bp
@@ -6085,14 +5892,13 @@ define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_passthru_v8i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI11_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z2.d, z1.d
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z3.d, z0.d
; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
@@ -6103,25 +5909,11 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
; VBITS_GE_256-NEXT: ptrue p1.d, vl1
; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
-; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
-; VBITS_GE_256-NEXT: umov w8, v2.b[0]
-; VBITS_GE_256-NEXT: umov w9, v2.b[1]
-; VBITS_GE_256-NEXT: umov w10, v2.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v2.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v2.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbnz w9, #0, .LBB11_10
+; VBITS_GE_256-NEXT: ldr q3, [x8, :lo12:.LCPI11_0]
+; VBITS_GE_256-NEXT: and v2.16b, v2.16b, v3.16b
+; VBITS_GE_256-NEXT: addv h2, v2.8h
+; VBITS_GE_256-NEXT: fmov w8, s2
+; VBITS_GE_256-NEXT: tbnz w8, #0, .LBB11_10
; VBITS_GE_256-NEXT: // %bb.1: // %else
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB11_11
; VBITS_GE_256-NEXT: .LBB11_2: // %else2
@@ -6148,7 +5940,6 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB11_10: // %cond.load
; VBITS_GE_256-NEXT: ldr x9, [x0], #8
@@ -6207,66 +5998,42 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_passthru_v8i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI11_0
+; VBITS_GE_512-NEXT: ldr q2, [x8, :lo12:.LCPI11_0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d
; VBITS_GE_512-NEXT: mov z1.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
-; VBITS_GE_512-NEXT: uzp1 z1.b, z1.b, z1.b
-; VBITS_GE_512-NEXT: umov w8, v1.b[0]
-; VBITS_GE_512-NEXT: umov w9, v1.b[1]
-; VBITS_GE_512-NEXT: umov w10, v1.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v1.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v1.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbnz w9, #0, .LBB11_10
+; VBITS_GE_512-NEXT: and v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: addv h1, v1.8h
+; VBITS_GE_512-NEXT: fmov w8, s1
+; VBITS_GE_512-NEXT: tbnz w8, #0, .LBB11_9
; VBITS_GE_512-NEXT: // %bb.1: // %else
-; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB11_11
+; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB11_10
; VBITS_GE_512-NEXT: .LBB11_2: // %else2
-; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB11_12
+; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB11_11
; VBITS_GE_512-NEXT: .LBB11_3: // %else6
-; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB11_13
+; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB11_12
; VBITS_GE_512-NEXT: .LBB11_4: // %else10
-; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB11_14
+; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB11_13
; VBITS_GE_512-NEXT: .LBB11_5: // %else14
-; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB11_15
+; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB11_14
; VBITS_GE_512-NEXT: .LBB11_6: // %else18
-; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB11_16
+; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB11_15
; VBITS_GE_512-NEXT: .LBB11_7: // %else22
-; VBITS_GE_512-NEXT: tbz w8, #7, .LBB11_9
-; VBITS_GE_512-NEXT: .LBB11_8: // %cond.load25
-; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
-; VBITS_GE_512-NEXT: index z1.d, #0, #1
-; VBITS_GE_512-NEXT: ptrue p1.d
-; VBITS_GE_512-NEXT: mov z2.d, x8
-; VBITS_GE_512-NEXT: ldr x8, [x0]
-; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
-; VBITS_GE_512-NEXT: mov z0.d, p2/m, x8
-; VBITS_GE_512-NEXT: .LBB11_9: // %else26
+; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB11_16
+; VBITS_GE_512-NEXT: .LBB11_8: // %else26
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
-; VBITS_GE_512-NEXT: .LBB11_10: // %cond.load
+; VBITS_GE_512-NEXT: .LBB11_9: // %cond.load
; VBITS_GE_512-NEXT: ldr x9, [x0], #8
; VBITS_GE_512-NEXT: ptrue p1.d, vl1
; VBITS_GE_512-NEXT: mov z0.d, p1/m, x9
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB11_2
-; VBITS_GE_512-NEXT: .LBB11_11: // %cond.load1
+; VBITS_GE_512-NEXT: .LBB11_10: // %cond.load1
; VBITS_GE_512-NEXT: mov w9, #1 // =0x1
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6275,7 +6042,7 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #2, .LBB11_3
-; VBITS_GE_512-NEXT: .LBB11_12: // %cond.load5
+; VBITS_GE_512-NEXT: .LBB11_11: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6284,7 +6051,7 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #3, .LBB11_4
-; VBITS_GE_512-NEXT: .LBB11_13: // %cond.load9
+; VBITS_GE_512-NEXT: .LBB11_12: // %cond.load9
; VBITS_GE_512-NEXT: mov w9, #3 // =0x3
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6293,7 +6060,7 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #4, .LBB11_5
-; VBITS_GE_512-NEXT: .LBB11_14: // %cond.load13
+; VBITS_GE_512-NEXT: .LBB11_13: // %cond.load13
; VBITS_GE_512-NEXT: mov w9, #4 // =0x4
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6302,7 +6069,7 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #5, .LBB11_6
-; VBITS_GE_512-NEXT: .LBB11_15: // %cond.load17
+; VBITS_GE_512-NEXT: .LBB11_14: // %cond.load17
; VBITS_GE_512-NEXT: mov w9, #5 // =0x5
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6311,7 +6078,7 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
; VBITS_GE_512-NEXT: tbz w8, #6, .LBB11_7
-; VBITS_GE_512-NEXT: .LBB11_16: // %cond.load21
+; VBITS_GE_512-NEXT: .LBB11_15: // %cond.load21
; VBITS_GE_512-NEXT: mov w9, #6 // =0x6
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6319,50 +6086,52 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr x9, [x0], #8
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: mov z0.d, p2/m, x9
-; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB11_8
-; VBITS_GE_512-NEXT: b .LBB11_9
+; VBITS_GE_512-NEXT: tbz w8, #7, .LBB11_8
+; VBITS_GE_512-NEXT: .LBB11_16: // %cond.load25
+; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
+; VBITS_GE_512-NEXT: index z1.d, #0, #1
+; VBITS_GE_512-NEXT: ptrue p1.d
+; VBITS_GE_512-NEXT: mov z2.d, x8
+; VBITS_GE_512-NEXT: ldr x8, [x0]
+; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
+; VBITS_GE_512-NEXT: mov z0.d, p2/m, x8
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
+; VBITS_GE_512-NEXT: ret
;
; CHECK-EXPAND-LABEL: masked_load_passthru_v8i64:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.d, vl4
-; CHECK-EXPAND-NEXT: mov x10, #4 // =0x4
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI11_0
+; CHECK-EXPAND-NEXT: ldr q2, [x8, :lo12:.LCPI11_0]
+; CHECK-EXPAND-NEXT: mov x8, #4 // =0x4
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1d { z1.d }, p0/z, [x1]
-; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x1, x10, lsl #3]
-; CHECK-EXPAND-NEXT: ld1d { z3.d }, p0/z, [x0, x10, lsl #3]
+; CHECK-EXPAND-NEXT: ld1d { z3.d }, p0/z, [x0, x8, lsl #3]
; CHECK-EXPAND-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
-; CHECK-EXPAND-NEXT: cmpeq p2.d, p0/z, z3.d, z2.d
; CHECK-EXPAND-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
-; CHECK-EXPAND-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-EXPAND-NEXT: umov w8, v0.h[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.h[1]
-; CHECK-EXPAND-NEXT: umov w11, v0.h[2]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.h[3]
-; CHECK-EXPAND-NEXT: bfi w8, w11, #2, #1
; CHECK-EXPAND-NEXT: cntp x11, p1, p1.d
-; CHECK-EXPAND-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.d
-; CHECK-EXPAND-NEXT: and w8, w8, #0xf
+; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-EXPAND-NEXT: whilelo p4.d, xzr, x11
-; CHECK-EXPAND-NEXT: fmov s0, w8
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
+; CHECK-EXPAND-NEXT: addv s0, v0.4s
+; CHECK-EXPAND-NEXT: cmpeq p2.d, p0/z, z3.d, z2.d
; CHECK-EXPAND-NEXT: ld1d { z3.d }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x9
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: fmov w9, s0
; CHECK-EXPAND-NEXT: expand z3.d, p1, z3.d
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.d
; CHECK-EXPAND-NEXT: mov z1.d, p1/m, z3.d
-; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x8, lsl #3]
+; CHECK-EXPAND-NEXT: and w9, w9, #0xf
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x9, lsl #3]
; CHECK-EXPAND-NEXT: st1d { z1.d }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.d, p2, z0.d
; CHECK-EXPAND-NEXT: sel z0.d, p2, z0.d, z2.d
-; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x10, lsl #3]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]
; CHECK-EXPAND-NEXT: ret
%a = load <8 x i64>, ptr %ap
%b = load <8 x i64>, ptr %bp
@@ -6375,14 +6144,13 @@ define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_passthru_v8f64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI12_0
; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z2.d, z1.d
; VBITS_GE_256-NEXT: fcmeq p2.d, p0/z, z3.d, z0.d
; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff
@@ -6392,25 +6160,11 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s
; VBITS_GE_256-NEXT: splice z3.s, p1, z3.s, z2.s
; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h
-; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b
-; VBITS_GE_256-NEXT: umov w8, v2.b[0]
-; VBITS_GE_256-NEXT: umov w9, v2.b[1]
-; VBITS_GE_256-NEXT: umov w10, v2.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v2.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v2.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v2.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbnz w9, #0, .LBB12_10
+; VBITS_GE_256-NEXT: ldr q3, [x8, :lo12:.LCPI12_0]
+; VBITS_GE_256-NEXT: and v2.16b, v2.16b, v3.16b
+; VBITS_GE_256-NEXT: addv h2, v2.8h
+; VBITS_GE_256-NEXT: fmov w8, s2
+; VBITS_GE_256-NEXT: tbnz w8, #0, .LBB12_10
; VBITS_GE_256-NEXT: // %bb.1: // %else
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB12_11
; VBITS_GE_256-NEXT: .LBB12_2: // %else2
@@ -6437,7 +6191,6 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB12_10: // %cond.load
; VBITS_GE_256-NEXT: ldr d2, [x0], #8
@@ -6498,66 +6251,42 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_passthru_v8f64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI12_0
+; VBITS_GE_512-NEXT: ldr q2, [x8, :lo12:.LCPI12_0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: fcmeq p1.d, p0/z, z1.d, z0.d
; VBITS_GE_512-NEXT: mov z1.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
-; VBITS_GE_512-NEXT: uzp1 z1.b, z1.b, z1.b
-; VBITS_GE_512-NEXT: umov w8, v1.b[0]
-; VBITS_GE_512-NEXT: umov w9, v1.b[1]
-; VBITS_GE_512-NEXT: umov w10, v1.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v1.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v1.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v1.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbnz w9, #0, .LBB12_10
+; VBITS_GE_512-NEXT: and v1.16b, v1.16b, v2.16b
+; VBITS_GE_512-NEXT: addv h1, v1.8h
+; VBITS_GE_512-NEXT: fmov w8, s1
+; VBITS_GE_512-NEXT: tbnz w8, #0, .LBB12_9
; VBITS_GE_512-NEXT: // %bb.1: // %else
-; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB12_11
+; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB12_10
; VBITS_GE_512-NEXT: .LBB12_2: // %else2
-; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB12_12
+; VBITS_GE_512-NEXT: tbnz w8, #2, .LBB12_11
; VBITS_GE_512-NEXT: .LBB12_3: // %else6
-; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB12_13
+; VBITS_GE_512-NEXT: tbnz w8, #3, .LBB12_12
; VBITS_GE_512-NEXT: .LBB12_4: // %else10
-; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB12_14
+; VBITS_GE_512-NEXT: tbnz w8, #4, .LBB12_13
; VBITS_GE_512-NEXT: .LBB12_5: // %else14
-; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB12_15
+; VBITS_GE_512-NEXT: tbnz w8, #5, .LBB12_14
; VBITS_GE_512-NEXT: .LBB12_6: // %else18
-; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB12_16
+; VBITS_GE_512-NEXT: tbnz w8, #6, .LBB12_15
; VBITS_GE_512-NEXT: .LBB12_7: // %else22
-; VBITS_GE_512-NEXT: tbz w8, #7, .LBB12_9
-; VBITS_GE_512-NEXT: .LBB12_8: // %cond.load25
-; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
-; VBITS_GE_512-NEXT: index z1.d, #0, #1
-; VBITS_GE_512-NEXT: ptrue p1.d
-; VBITS_GE_512-NEXT: mov z2.d, x8
-; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
-; VBITS_GE_512-NEXT: ldr d1, [x0]
-; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
-; VBITS_GE_512-NEXT: .LBB12_9: // %else26
+; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB12_16
+; VBITS_GE_512-NEXT: .LBB12_8: // %else26
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
-; VBITS_GE_512-NEXT: .LBB12_10: // %cond.load
+; VBITS_GE_512-NEXT: .LBB12_9: // %cond.load
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: ptrue p1.d, vl1
; VBITS_GE_512-NEXT: mov z0.d, p1/m, z1.d
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB12_2
-; VBITS_GE_512-NEXT: .LBB12_11: // %cond.load1
+; VBITS_GE_512-NEXT: .LBB12_10: // %cond.load1
; VBITS_GE_512-NEXT: mov w9, #1 // =0x1
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6566,7 +6295,7 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
; VBITS_GE_512-NEXT: tbz w8, #2, .LBB12_3
-; VBITS_GE_512-NEXT: .LBB12_12: // %cond.load5
+; VBITS_GE_512-NEXT: .LBB12_11: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6575,7 +6304,7 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
; VBITS_GE_512-NEXT: tbz w8, #3, .LBB12_4
-; VBITS_GE_512-NEXT: .LBB12_13: // %cond.load9
+; VBITS_GE_512-NEXT: .LBB12_12: // %cond.load9
; VBITS_GE_512-NEXT: mov w9, #3 // =0x3
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6584,7 +6313,7 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
; VBITS_GE_512-NEXT: tbz w8, #4, .LBB12_5
-; VBITS_GE_512-NEXT: .LBB12_14: // %cond.load13
+; VBITS_GE_512-NEXT: .LBB12_13: // %cond.load13
; VBITS_GE_512-NEXT: mov w9, #4 // =0x4
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6593,7 +6322,7 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
; VBITS_GE_512-NEXT: tbz w8, #5, .LBB12_6
-; VBITS_GE_512-NEXT: .LBB12_15: // %cond.load17
+; VBITS_GE_512-NEXT: .LBB12_14: // %cond.load17
; VBITS_GE_512-NEXT: mov w9, #5 // =0x5
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6602,7 +6331,7 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
; VBITS_GE_512-NEXT: tbz w8, #6, .LBB12_7
-; VBITS_GE_512-NEXT: .LBB12_16: // %cond.load21
+; VBITS_GE_512-NEXT: .LBB12_15: // %cond.load21
; VBITS_GE_512-NEXT: mov w9, #6 // =0x6
; VBITS_GE_512-NEXT: index z1.d, #0, #1
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -6610,50 +6339,52 @@ define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
; VBITS_GE_512-NEXT: ldr d1, [x0], #8
; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
-; VBITS_GE_512-NEXT: tbnz w8, #7, .LBB12_8
-; VBITS_GE_512-NEXT: b .LBB12_9
+; VBITS_GE_512-NEXT: tbz w8, #7, .LBB12_8
+; VBITS_GE_512-NEXT: .LBB12_16: // %cond.load25
+; VBITS_GE_512-NEXT: mov w8, #7 // =0x7
+; VBITS_GE_512-NEXT: index z1.d, #0, #1
+; VBITS_GE_512-NEXT: ptrue p1.d
+; VBITS_GE_512-NEXT: mov z2.d, x8
+; VBITS_GE_512-NEXT: cmpeq p2.d, p1/z, z1.d, z2.d
+; VBITS_GE_512-NEXT: ldr d1, [x0]
+; VBITS_GE_512-NEXT: mov z0.d, p2/m, d1
+; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
+; VBITS_GE_512-NEXT: ret
;
; CHECK-EXPAND-LABEL: masked_load_passthru_v8f64:
; CHECK-EXPAND: // %bb.0:
-; CHECK-EXPAND-NEXT: sub sp, sp, #16
-; CHECK-EXPAND-NEXT: .cfi_def_cfa_offset 16
; CHECK-EXPAND-NEXT: ptrue p0.d, vl4
-; CHECK-EXPAND-NEXT: mov x10, #4 // =0x4
+; CHECK-EXPAND-NEXT: adrp x8, .LCPI12_0
+; CHECK-EXPAND-NEXT: ldr q2, [x8, :lo12:.LCPI12_0]
+; CHECK-EXPAND-NEXT: mov x8, #4 // =0x4
; CHECK-EXPAND-NEXT: ptrue p3.s
; CHECK-EXPAND-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-EXPAND-NEXT: ld1d { z1.d }, p0/z, [x1]
-; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x1, x10, lsl #3]
-; CHECK-EXPAND-NEXT: ld1d { z3.d }, p0/z, [x0, x10, lsl #3]
+; CHECK-EXPAND-NEXT: ld1d { z3.d }, p0/z, [x0, x8, lsl #3]
; CHECK-EXPAND-NEXT: fcmeq p1.d, p0/z, z0.d, z1.d
-; CHECK-EXPAND-NEXT: fcmeq p2.d, p0/z, z3.d, z2.d
; CHECK-EXPAND-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
-; CHECK-EXPAND-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-EXPAND-NEXT: umov w8, v0.h[0]
-; CHECK-EXPAND-NEXT: umov w9, v0.h[1]
-; CHECK-EXPAND-NEXT: umov w11, v0.h[2]
-; CHECK-EXPAND-NEXT: and w8, w8, #0x1
-; CHECK-EXPAND-NEXT: bfi w8, w9, #1, #1
-; CHECK-EXPAND-NEXT: umov w9, v0.h[3]
-; CHECK-EXPAND-NEXT: bfi w8, w11, #2, #1
; CHECK-EXPAND-NEXT: cntp x11, p1, p1.d
-; CHECK-EXPAND-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-EXPAND-NEXT: cntp x9, p2, p2.d
-; CHECK-EXPAND-NEXT: and w8, w8, #0xf
+; CHECK-EXPAND-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-EXPAND-NEXT: whilelo p4.d, xzr, x11
-; CHECK-EXPAND-NEXT: fmov s0, w8
+; CHECK-EXPAND-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-EXPAND-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3]
+; CHECK-EXPAND-NEXT: addv s0, v0.4s
+; CHECK-EXPAND-NEXT: fcmeq p2.d, p0/z, z3.d, z2.d
; CHECK-EXPAND-NEXT: ld1d { z3.d }, p4/z, [x0]
-; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
-; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x9
-; CHECK-EXPAND-NEXT: fmov w8, s0
+; CHECK-EXPAND-NEXT: fmov w9, s0
; CHECK-EXPAND-NEXT: expand z3.d, p1, z3.d
+; CHECK-EXPAND-NEXT: cntp x10, p2, p2.d
; CHECK-EXPAND-NEXT: mov z1.d, p1/m, z3.d
-; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x8, lsl #3]
+; CHECK-EXPAND-NEXT: and w9, w9, #0xf
+; CHECK-EXPAND-NEXT: fmov s0, w9
+; CHECK-EXPAND-NEXT: cnt z0.s, p3/z, z0.s
+; CHECK-EXPAND-NEXT: whilelo p3.d, xzr, x10
+; CHECK-EXPAND-NEXT: fmov w9, s0
+; CHECK-EXPAND-NEXT: ld1d { z0.d }, p3/z, [x0, x9, lsl #3]
; CHECK-EXPAND-NEXT: st1d { z1.d }, p0, [x2]
; CHECK-EXPAND-NEXT: expand z0.d, p2, z0.d
; CHECK-EXPAND-NEXT: sel z0.d, p2, z0.d, z2.d
-; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x10, lsl #3]
-; CHECK-EXPAND-NEXT: add sp, sp, #16
+; CHECK-EXPAND-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]
; CHECK-EXPAND-NEXT: ret
%a = load <8 x double>, ptr %ap
%b = load <8 x double>, ptr %bp
@@ -8003,64 +7734,28 @@ define void @masked_load_sext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v16i16i32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.h, vl16
+; VBITS_GE_256-NEXT: adrp x8, .LCPI16_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpeq p0.h, p1/z, z0.h, #0
; VBITS_GE_256-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.h
; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: umov w11, v0.b[7]
-; VBITS_GE_256-NEXT: umov w12, v0.b[8]
-; VBITS_GE_256-NEXT: umov w13, v0.b[3]
-; VBITS_GE_256-NEXT: umov w14, v0.b[4]
-; VBITS_GE_256-NEXT: umov w15, v0.b[10]
-; VBITS_GE_256-NEXT: umov w16, v0.b[5]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[9]
-; VBITS_GE_256-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_256-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_256-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[11]
-; VBITS_GE_256-NEXT: orr w11, w11, w12
-; VBITS_GE_256-NEXT: umov w12, v0.b[13]
-; VBITS_GE_256-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_256-NEXT: umov w13, v0.b[12]
-; VBITS_GE_256-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_256-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_256-NEXT: umov w14, v0.b[14]
-; VBITS_GE_256-NEXT: orr w9, w11, w9
-; VBITS_GE_256-NEXT: umov w11, v0.b[6]
-; VBITS_GE_256-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w15
-; VBITS_GE_256-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_256-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w10
-; VBITS_GE_256-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w13
-; VBITS_GE_256-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_256-NEXT: umov w13, v0.b[15]
-; VBITS_GE_256-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w10
-; VBITS_GE_256-NEXT: orr w9, w9, w12
-; VBITS_GE_256-NEXT: orr w8, w8, w9
-; VBITS_GE_256-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_256-NEXT: and w8, w9, #0xffff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB16_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB16_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #2
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB16_3
; VBITS_GE_256-NEXT: b .LBB16_4
; VBITS_GE_256-NEXT: .LBB16_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI16_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI16_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI16_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI16_1
; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB16_4
; VBITS_GE_256-NEXT: .LBB16_3: // %cond.load1
@@ -8114,7 +7809,6 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB16_20: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -8224,64 +7918,28 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.h, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI16_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p0.h, p1/z, z0.h, #0
; VBITS_GE_512-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB16_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB16_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rh { z0.h }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #2
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB16_3
; VBITS_GE_512-NEXT: b .LBB16_4
; VBITS_GE_512-NEXT: .LBB16_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI16_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI16_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI16_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI16_1
; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB16_4
; VBITS_GE_512-NEXT: .LBB16_3: // %cond.load1
@@ -8330,7 +7988,6 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB16_20: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -8614,41 +8271,26 @@ define void @masked_load_sext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v8i32i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: adrp x8, .LCPI18_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI18_0]
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpeq p0.s, p1/z, z0.s, #0
; VBITS_GE_256-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB18_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB18_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB18_3
; VBITS_GE_256-NEXT: b .LBB18_4
; VBITS_GE_256-NEXT: .LBB18_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI18_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI18_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI18_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI18_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB18_4
; VBITS_GE_256-NEXT: .LBB18_3: // %cond.load1
@@ -8686,7 +8328,6 @@ define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB18_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -8732,41 +8373,26 @@ define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.s, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI18_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI18_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p0.s, p1/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB18_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB18_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB18_3
; VBITS_GE_512-NEXT: b .LBB18_4
; VBITS_GE_512-NEXT: .LBB18_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI18_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI18_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI18_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI18_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB18_4
; VBITS_GE_512-NEXT: .LBB18_3: // %cond.load1
@@ -8799,7 +8425,6 @@ define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB18_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -10209,64 +9834,28 @@ define void @masked_load_zext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v16i16i32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.h, vl16
+; VBITS_GE_256-NEXT: adrp x8, .LCPI22_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI22_0]
; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpeq p0.h, p1/z, z0.h, #0
; VBITS_GE_256-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.h
; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: umov w11, v0.b[7]
-; VBITS_GE_256-NEXT: umov w12, v0.b[8]
-; VBITS_GE_256-NEXT: umov w13, v0.b[3]
-; VBITS_GE_256-NEXT: umov w14, v0.b[4]
-; VBITS_GE_256-NEXT: umov w15, v0.b[10]
-; VBITS_GE_256-NEXT: umov w16, v0.b[5]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[9]
-; VBITS_GE_256-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_256-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_256-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[11]
-; VBITS_GE_256-NEXT: orr w11, w11, w12
-; VBITS_GE_256-NEXT: umov w12, v0.b[13]
-; VBITS_GE_256-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_256-NEXT: umov w13, v0.b[12]
-; VBITS_GE_256-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_256-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_256-NEXT: umov w14, v0.b[14]
-; VBITS_GE_256-NEXT: orr w9, w11, w9
-; VBITS_GE_256-NEXT: umov w11, v0.b[6]
-; VBITS_GE_256-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w15
-; VBITS_GE_256-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_256-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w10
-; VBITS_GE_256-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w13
-; VBITS_GE_256-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_256-NEXT: umov w13, v0.b[15]
-; VBITS_GE_256-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w9, w10
-; VBITS_GE_256-NEXT: orr w9, w9, w12
-; VBITS_GE_256-NEXT: orr w8, w8, w9
-; VBITS_GE_256-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_256-NEXT: and w8, w9, #0xffff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB22_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_256-NEXT: umov w8, v0.h[0]
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB22_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #2
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB22_3
; VBITS_GE_256-NEXT: b .LBB22_4
; VBITS_GE_256-NEXT: .LBB22_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI22_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI22_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI22_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI22_1
; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB22_4
; VBITS_GE_256-NEXT: .LBB22_3: // %cond.load1
@@ -10320,7 +9909,6 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB22_20: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -10430,64 +10018,28 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.h, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI22_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI22_0]
; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p0.h, p1/z, z0.h, #0
; VBITS_GE_512-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB22_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB22_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rh { z0.h }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #2
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB22_3
; VBITS_GE_512-NEXT: b .LBB22_4
; VBITS_GE_512-NEXT: .LBB22_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI22_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI22_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI22_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI22_1
; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB22_4
; VBITS_GE_512-NEXT: .LBB22_3: // %cond.load1
@@ -10536,7 +10088,6 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB22_20: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -10820,41 +10371,26 @@ define void @masked_load_zext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v8i32i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: adrp x8, .LCPI24_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI24_0]
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpeq p0.s, p1/z, z0.s, #0
; VBITS_GE_256-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB24_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB24_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB24_3
; VBITS_GE_256-NEXT: b .LBB24_4
; VBITS_GE_256-NEXT: .LBB24_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI24_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI24_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI24_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI24_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB24_4
; VBITS_GE_256-NEXT: .LBB24_3: // %cond.load1
@@ -10892,7 +10428,6 @@ define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB24_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -10938,41 +10473,26 @@ define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.s, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI24_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI24_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p0.s, p1/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB24_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB24_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB24_3
; VBITS_GE_512-NEXT: b .LBB24_4
; VBITS_GE_512-NEXT: .LBB24_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI24_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI24_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI24_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI24_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB24_4
; VBITS_GE_512-NEXT: .LBB24_3: // %cond.load1
@@ -11005,7 +10525,6 @@ define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB24_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -12012,8 +11531,6 @@ define void @masked_load_sext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v16i8i32_m32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
@@ -12033,9 +11550,8 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB26_2
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB26_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -12085,7 +11601,6 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB26_20: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -12130,56 +11645,20 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v16i8i32_m32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI26_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI26_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB26_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB26_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -12224,7 +11703,6 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b
; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB26_20: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -12307,12 +11785,11 @@ define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v8i8i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI27_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -12322,25 +11799,11 @@ define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB27_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI27_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB27_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -12375,7 +11838,6 @@ define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB27_12: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -12396,33 +11858,18 @@ define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v8i8i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI27_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI27_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB27_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB27_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -12452,7 +11899,6 @@ define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB27_12: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -12513,8 +11959,6 @@ define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v16i16i32_m32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
@@ -12535,9 +11979,8 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB28_2
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB28_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p1/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #2
@@ -12599,7 +12042,6 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB28_20: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -12709,57 +12151,21 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32_m32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI28_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI28_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.h
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB28_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB28_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rh { z0.h }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #2
@@ -12767,8 +12173,8 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: b .LBB28_4
; VBITS_GE_512-NEXT: .LBB28_2:
; VBITS_GE_512-NEXT: ptrue p2.h, vl16
-; VBITS_GE_512-NEXT: adrp x9, .LCPI28_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI28_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI28_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI28_1
; VBITS_GE_512-NEXT: ld1h { z0.h }, p2/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB28_4
; VBITS_GE_512-NEXT: .LBB28_3: // %cond.load1
@@ -12816,7 +12222,6 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: .LBB28_19: // %else58
; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB28_20: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -12964,12 +12369,11 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v8i16i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI29_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -12979,25 +12383,11 @@ define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB29_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI29_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB29_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrh w9, [x0], #2
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -13031,7 +12421,6 @@ define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB29_12: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.h }[2], [x0], #2
@@ -13052,33 +12441,18 @@ define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v8i16i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI29_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI29_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB29_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB29_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrh w9, [x0], #2
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -13107,7 +12481,6 @@ define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB29_12: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.h }[2], [x0], #2
@@ -13166,12 +12539,11 @@ define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_v8i32i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI30_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -13182,25 +12554,11 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: ptrue p1.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB30_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI30_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB30_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p1/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
@@ -13208,8 +12566,8 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: b .LBB30_4
; VBITS_GE_256-NEXT: .LBB30_2:
; VBITS_GE_256-NEXT: ptrue p2.s, vl8
-; VBITS_GE_256-NEXT: adrp x9, .LCPI30_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI30_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI30_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI30_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p2/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB30_4
; VBITS_GE_256-NEXT: .LBB30_3: // %cond.load1
@@ -13246,7 +12604,6 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB30_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -13292,34 +12649,19 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI30_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI30_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.s
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB30_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB30_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
@@ -13327,8 +12669,8 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: b .LBB30_4
; VBITS_GE_512-NEXT: .LBB30_2:
; VBITS_GE_512-NEXT: ptrue p2.s, vl8
-; VBITS_GE_512-NEXT: adrp x9, .LCPI30_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI30_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI30_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI30_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p2/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB30_4
; VBITS_GE_512-NEXT: .LBB30_3: // %cond.load1
@@ -13360,7 +12702,6 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: .LBB30_11: // %else26
; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB30_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -14376,8 +13717,6 @@ define void @masked_load_zext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v16i8i32_m32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
@@ -14397,9 +13736,8 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB32_2
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB32_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -14449,7 +13787,6 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB32_20: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -14494,56 +13831,20 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v16i8i32_m32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI32_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI32_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB32_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB32_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -14588,7 +13889,6 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB32_20: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -14671,12 +13971,11 @@ define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v8i8i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI33_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -14686,25 +13985,11 @@ define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB33_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI33_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB33_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrb w9, [x0], #1
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -14739,7 +14024,6 @@ define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB33_12: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -14760,33 +14044,18 @@ define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v8i8i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI33_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI33_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB33_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB33_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrb w9, [x0], #1
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -14816,7 +14085,6 @@ define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB33_12: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.b }[2], [x0], #1
@@ -14877,8 +14145,6 @@ define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v16i16i32_m32:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.s, vl8
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]
@@ -14899,9 +14165,8 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
; VBITS_GE_256-NEXT: addp v0.16b, v0.16b, v0.16b
-; VBITS_GE_256-NEXT: umov w9, v0.h[0]
; VBITS_GE_256-NEXT: umov w8, v0.h[0]
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB34_2
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB34_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rh { z0.h }, p1/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #2
@@ -14963,7 +14228,6 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB34_20: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -15073,57 +14337,21 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32_m32:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512-NEXT: adrp x8, .LCPI34_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI34_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.h
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: umov w11, v0.b[7]
-; VBITS_GE_512-NEXT: umov w12, v0.b[8]
-; VBITS_GE_512-NEXT: umov w13, v0.b[3]
-; VBITS_GE_512-NEXT: umov w14, v0.b[4]
-; VBITS_GE_512-NEXT: umov w15, v0.b[10]
-; VBITS_GE_512-NEXT: umov w16, v0.b[5]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[9]
-; VBITS_GE_512-NEXT: ubfiz w11, w11, #7, #1
-; VBITS_GE_512-NEXT: ubfiz w12, w12, #8, #1
-; VBITS_GE_512-NEXT: ubfiz w15, w15, #10, #1
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[11]
-; VBITS_GE_512-NEXT: orr w11, w11, w12
-; VBITS_GE_512-NEXT: umov w12, v0.b[13]
-; VBITS_GE_512-NEXT: bfi w8, w13, #3, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[12]
-; VBITS_GE_512-NEXT: ubfiz w9, w9, #9, #1
-; VBITS_GE_512-NEXT: bfi w8, w14, #4, #1
-; VBITS_GE_512-NEXT: umov w14, v0.b[14]
-; VBITS_GE_512-NEXT: orr w9, w11, w9
-; VBITS_GE_512-NEXT: umov w11, v0.b[6]
-; VBITS_GE_512-NEXT: ubfiz w10, w10, #11, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w15
-; VBITS_GE_512-NEXT: ubfiz w13, w13, #12, #1
-; VBITS_GE_512-NEXT: bfi w8, w16, #5, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: ubfiz w10, w12, #13, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w13
-; VBITS_GE_512-NEXT: ubfiz w12, w14, #14, #1
-; VBITS_GE_512-NEXT: umov w13, v0.b[15]
-; VBITS_GE_512-NEXT: bfi w8, w11, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w9, w10
-; VBITS_GE_512-NEXT: orr w9, w9, w12
-; VBITS_GE_512-NEXT: orr w8, w8, w9
-; VBITS_GE_512-NEXT: orr w9, w8, w13, lsl #15
-; VBITS_GE_512-NEXT: and w8, w9, #0xffff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB34_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: addp v0.16b, v0.16b, v0.16b
+; VBITS_GE_512-NEXT: umov w8, v0.h[0]
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB34_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rh { z0.h }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #2
@@ -15131,8 +14359,8 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: b .LBB34_4
; VBITS_GE_512-NEXT: .LBB34_2:
; VBITS_GE_512-NEXT: ptrue p2.h, vl16
-; VBITS_GE_512-NEXT: adrp x9, .LCPI34_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI34_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI34_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI34_1
; VBITS_GE_512-NEXT: ld1h { z0.h }, p2/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB34_4
; VBITS_GE_512-NEXT: .LBB34_3: // %cond.load1
@@ -15180,7 +14408,6 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: .LBB34_19: // %else58
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB34_20: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -15328,12 +14555,11 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v8i16i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI35_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -15343,25 +14569,11 @@ define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB35_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI35_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB35_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ldrh w9, [x0], #2
; VBITS_GE_256-NEXT: fmov s0, w9
@@ -15395,7 +14607,6 @@ define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB35_12: // %cond.load5
; VBITS_GE_256-NEXT: ld1 { v0.h }[2], [x0], #2
@@ -15416,33 +14627,18 @@ define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v8i16i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI35_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI35_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB35_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB35_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ldrh w9, [x0], #2
; VBITS_GE_512-NEXT: fmov s0, w9
@@ -15471,7 +14667,6 @@ define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB35_12: // %cond.load5
; VBITS_GE_512-NEXT: ld1 { v0.h }[2], [x0], #2
@@ -15530,12 +14725,11 @@ define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_v8i32i64_m64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT: adrp x8, .LCPI36_0
; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0
; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
@@ -15546,25 +14740,11 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s
; VBITS_GE_256-NEXT: ptrue p1.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB36_2
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI36_0]
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB36_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p1/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
@@ -15572,8 +14752,8 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: b .LBB36_4
; VBITS_GE_256-NEXT: .LBB36_2:
; VBITS_GE_256-NEXT: ptrue p2.s, vl8
-; VBITS_GE_256-NEXT: adrp x9, .LCPI36_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI36_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI36_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI36_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p2/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB36_4
; VBITS_GE_256-NEXT: .LBB36_3: // %cond.load1
@@ -15610,7 +14790,6 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB36_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -15656,34 +14835,19 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64_m64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI36_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI36_0]
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0
; VBITS_GE_512-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p1.s
; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB36_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB36_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p1/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
@@ -15691,8 +14855,8 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: b .LBB36_4
; VBITS_GE_512-NEXT: .LBB36_2:
; VBITS_GE_512-NEXT: ptrue p2.s, vl8
-; VBITS_GE_512-NEXT: adrp x9, .LCPI36_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI36_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI36_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI36_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p2/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB36_4
; VBITS_GE_512-NEXT: .LBB36_3: // %cond.load1
@@ -15724,7 +14888,6 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: .LBB36_11: // %else26
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB36_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -26090,41 +25253,26 @@ define void @masked_load_zext_v32i32i64(ptr %ap, ptr %bp, ptr %c) vscale_range(1
define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_sext_ugt_v8i32i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: adrp x8, .LCPI49_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI49_0]
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpne p0.s, p1/z, z0.s, #0
; VBITS_GE_256-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB49_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB49_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB49_3
; VBITS_GE_256-NEXT: b .LBB49_4
; VBITS_GE_256-NEXT: .LBB49_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI49_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI49_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI49_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI49_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB49_4
; VBITS_GE_256-NEXT: .LBB49_3: // %cond.load1
@@ -26162,7 +25310,6 @@ define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB49_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -26208,41 +25355,26 @@ define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_sext_ugt_v8i32i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.s, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI49_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI49_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpne p0.s, p1/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB49_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB49_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB49_3
; VBITS_GE_512-NEXT: b .LBB49_4
; VBITS_GE_512-NEXT: .LBB49_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI49_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI49_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI49_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI49_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB49_4
; VBITS_GE_512-NEXT: .LBB49_3: // %cond.load1
@@ -26275,7 +25407,6 @@ define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB49_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
@@ -26348,41 +25479,26 @@ define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-LABEL: masked_load_zext_sgt_v8i32i64:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: sub sp, sp, #16
-; VBITS_GE_256-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_256-NEXT: ptrue p1.s, vl8
+; VBITS_GE_256-NEXT: adrp x8, .LCPI50_0
+; VBITS_GE_256-NEXT: ldr q1, [x8, :lo12:.LCPI50_0]
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_256-NEXT: cmpgt p0.s, p1/z, z0.s, #0
; VBITS_GE_256-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_256-NEXT: ptrue p0.s
; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT: umov w8, v0.b[0]
-; VBITS_GE_256-NEXT: umov w9, v0.b[1]
-; VBITS_GE_256-NEXT: umov w10, v0.b[2]
-; VBITS_GE_256-NEXT: and w8, w8, #0x1
-; VBITS_GE_256-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[3]
-; VBITS_GE_256-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[4]
-; VBITS_GE_256-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[5]
-; VBITS_GE_256-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_256-NEXT: umov w10, v0.b[6]
-; VBITS_GE_256-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_256-NEXT: umov w9, v0.b[7]
-; VBITS_GE_256-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_256-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_256-NEXT: and w8, w9, #0xff
-; VBITS_GE_256-NEXT: tbz w9, #0, .LBB50_2
+; VBITS_GE_256-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_256-NEXT: addv h0, v0.8h
+; VBITS_GE_256-NEXT: fmov w8, s0
+; VBITS_GE_256-NEXT: tbz w8, #0, .LBB50_2
; VBITS_GE_256-NEXT: // %bb.1: // %cond.load
; VBITS_GE_256-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_256-NEXT: add x0, x0, #4
; VBITS_GE_256-NEXT: tbnz w8, #1, .LBB50_3
; VBITS_GE_256-NEXT: b .LBB50_4
; VBITS_GE_256-NEXT: .LBB50_2:
-; VBITS_GE_256-NEXT: adrp x9, .LCPI50_0
-; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI50_0
+; VBITS_GE_256-NEXT: adrp x9, .LCPI50_1
+; VBITS_GE_256-NEXT: add x9, x9, :lo12:.LCPI50_1
; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_256-NEXT: tbz w8, #1, .LBB50_4
; VBITS_GE_256-NEXT: .LBB50_3: // %cond.load1
@@ -26420,7 +25536,6 @@ define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]
-; VBITS_GE_256-NEXT: add sp, sp, #16
; VBITS_GE_256-NEXT: ret
; VBITS_GE_256-NEXT: .LBB50_12: // %cond.load5
; VBITS_GE_256-NEXT: mov w9, #2 // =0x2
@@ -26466,41 +25581,26 @@ define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
;
; VBITS_GE_512-LABEL: masked_load_zext_sgt_v8i32i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: sub sp, sp, #16
-; VBITS_GE_512-NEXT: .cfi_def_cfa_offset 16
; VBITS_GE_512-NEXT: ptrue p1.s, vl8
+; VBITS_GE_512-NEXT: adrp x8, .LCPI50_0
+; VBITS_GE_512-NEXT: ldr q1, [x8, :lo12:.LCPI50_0]
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x1]
; VBITS_GE_512-NEXT: cmpgt p0.s, p1/z, z0.s, #0
; VBITS_GE_512-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff
; VBITS_GE_512-NEXT: ptrue p0.s
; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
-; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b
-; VBITS_GE_512-NEXT: umov w8, v0.b[0]
-; VBITS_GE_512-NEXT: umov w9, v0.b[1]
-; VBITS_GE_512-NEXT: umov w10, v0.b[2]
-; VBITS_GE_512-NEXT: and w8, w8, #0x1
-; VBITS_GE_512-NEXT: bfi w8, w9, #1, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[3]
-; VBITS_GE_512-NEXT: bfi w8, w10, #2, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[4]
-; VBITS_GE_512-NEXT: bfi w8, w9, #3, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[5]
-; VBITS_GE_512-NEXT: bfi w8, w10, #4, #1
-; VBITS_GE_512-NEXT: umov w10, v0.b[6]
-; VBITS_GE_512-NEXT: bfi w8, w9, #5, #1
-; VBITS_GE_512-NEXT: umov w9, v0.b[7]
-; VBITS_GE_512-NEXT: bfi w8, w10, #6, #1
-; VBITS_GE_512-NEXT: orr w9, w8, w9, lsl #7
-; VBITS_GE_512-NEXT: and w8, w9, #0xff
-; VBITS_GE_512-NEXT: tbz w9, #0, .LBB50_2
+; VBITS_GE_512-NEXT: and v0.16b, v0.16b, v1.16b
+; VBITS_GE_512-NEXT: addv h0, v0.8h
+; VBITS_GE_512-NEXT: fmov w8, s0
+; VBITS_GE_512-NEXT: tbz w8, #0, .LBB50_2
; VBITS_GE_512-NEXT: // %bb.1: // %cond.load
; VBITS_GE_512-NEXT: ld1rw { z0.s }, p0/z, [x0]
; VBITS_GE_512-NEXT: add x0, x0, #4
; VBITS_GE_512-NEXT: tbnz w8, #1, .LBB50_3
; VBITS_GE_512-NEXT: b .LBB50_4
; VBITS_GE_512-NEXT: .LBB50_2:
-; VBITS_GE_512-NEXT: adrp x9, .LCPI50_0
-; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI50_0
+; VBITS_GE_512-NEXT: adrp x9, .LCPI50_1
+; VBITS_GE_512-NEXT: add x9, x9, :lo12:.LCPI50_1
; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x9]
; VBITS_GE_512-NEXT: tbz w8, #1, .LBB50_4
; VBITS_GE_512-NEXT: .LBB50_3: // %cond.load1
@@ -26533,7 +25633,6 @@ define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]
-; VBITS_GE_512-NEXT: add sp, sp, #16
; VBITS_GE_512-NEXT: ret
; VBITS_GE_512-NEXT: .LBB50_12: // %cond.load5
; VBITS_GE_512-NEXT: mov w9, #2 // =0x2
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
new file mode 100644
index 0000000000000..d968265b24136
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+; Bitmasks (bitcast <N x i1> to iN) from masks legalized out of >128-bit
+; sources. These all lower by narrowing the per-lane width until the mask fits
+; a 128-bit vector (the lane count, and so the bitmask, is unchanged), then
+; reusing the register-only compare->bitmask lowering. None of them may touch
+; the stack: the old lowering bailed here and fell into the generic
+; bitcast-through-memory legalization, leaving behind a dead frame.
+
+; 256-bit source, 4 lanes: narrowed <4 x i64> -> <4 x i32>.
+define i4 @bitmask_v4i64(<4 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v4i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI0_0
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv s0, v0.4s
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v4i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI0_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <4 x i64> %v, zeroinitializer
+ %b = bitcast <4 x i1> %c to i4
+ ret i4 %b
+}
+
+; 256-bit source, 8 lanes: narrowed <8 x i32> -> <8 x i16>.
+define i8 @bitmask_v8i32(<8 x i32> %v) {
+; CHECK-LE-LABEL: bitmask_v8i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI1_0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI1_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i32> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
+
+; 512-bit source, 8 lanes: narrowed <8 x i64> -> <8 x i16>.
+define i8 @bitmask_v8i64(<8 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v8i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI2_0
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI2_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i64> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
+
+; 512-bit source, 16 lanes: narrowed <16 x i32> -> <16 x i8>.
+define i16 @bitmask_v16i32(<16 x i32> %v) {
+; CHECK-LE-LABEL: bitmask_v16i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v3.4s, v3.4s, #0
+; CHECK-LE-NEXT: cmeq v2.4s, v2.4s, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI3_0
+; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w0, v0.h[0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v16i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI3_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v3.4s, v3.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmeq v2.4s, v2.4s, #0
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <16 x i32> %v, zeroinitializer
+ %b = bitcast <16 x i1> %c to i16
+ ret i16 %b
+}
+
+; 1024-bit source, 16 lanes: narrowed <16 x i64> -> <16 x i8> (maximum
+; narrowing ratio: 8x).
+define i16 @bitmask_v16i64(<16 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v16i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v7.2d, v7.2d, #0
+; CHECK-LE-NEXT: cmeq v6.2d, v6.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI4_0
+; CHECK-LE-NEXT: cmeq v5.2d, v5.2d, #0
+; CHECK-LE-NEXT: cmeq v4.2d, v4.2d, #0
+; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-LE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-LE-NEXT: mvn v1.16b, v1.16b
+; CHECK-LE-NEXT: mvn v0.16b, v0.16b
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w0, v0.h[0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v16i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v6.16b, v6.16b, v6.16b, #8
+; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI4_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI4_0
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
+; CHECK-BE-NEXT: ext v5.16b, v5.16b, v5.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v7.2d, v7.2d, #0
+; CHECK-BE-NEXT: cmeq v6.2d, v6.2d, #0
+; CHECK-BE-NEXT: cmeq v4.2d, v4.2d, #0
+; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-BE-NEXT: cmeq v5.2d, v5.2d, #0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-BE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-BE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BE-NEXT: mvn v1.16b, v1.16b
+; CHECK-BE-NEXT: mvn v0.16b, v0.16b
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <16 x i64> %v, zeroinitializer
+ %b = bitcast <16 x i1> %c to i16
+ ret i16 %b
+}
+
+; Float compares from a wide source take the same path.
+define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LE-LABEL: bitmask_v8f32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-LE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-LE-NEXT: adrp x8, .LCPI5_0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8f32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI5_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI5_0
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-BE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = fcmp oeq <8 x float> %a, %b
+ %m = bitcast <8 x i1> %c to i8
+ ret i8 %m
+}
+
+; With SVE the wide compare stays in a predicate; the bitmask is still built
+; register-only (predicate -> Z register -> narrow -> weights+reduce).
+; TODO: a predicated UADDV over the weight vector would shorten this further.
+define i8 @bitmask_v8i32_sve(<8 x i32> %v) vscale_range(2,2) "target-features"="+sve" {
+; CHECK-LE-LABEL: bitmask_v8i32_sve:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ptrue p0.s, vl4
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-LE-NEXT: adrp x8, .LCPI6_0
+; CHECK-LE-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-LE-NEXT: ptrue p0.s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI6_0]
+; CHECK-LE-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-LE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-LE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i32_sve:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI6_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
+; CHECK-BE-NEXT: ptrue p0.s, vl4
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-BE-NEXT: ptrue p0.s
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-BE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-BE-NEXT: revb z0.s, p0/m, z0.s
+; CHECK-BE-NEXT: ptrue p0.h
+; CHECK-BE-NEXT: revb z0.h, p0/m, z0.h
+; CHECK-BE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i32> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index f9ecae443d399..31429213cd2cd 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -740,8 +740,6 @@ define i4 @convert_to_bitmask_float(<4 x float> %vec) {
define i8 @convert_large_vector(<8 x i32> %vec) {
; CHECK-SD-LABEL: convert_large_vector:
; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: sub sp, sp, #16
-; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: cmeq.4s v1, v1, #0
; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
; CHECK-SD-NEXT: adrp x8, lCPI15_0 at PAGE
@@ -749,9 +747,7 @@ define i8 @convert_large_vector(<8 x i32> %vec) {
; CHECK-SD-NEXT: ldr q1, [x8, lCPI15_0 at PAGEOFF]
; CHECK-SD-NEXT: bic.16b v0, v1, v0
; CHECK-SD-NEXT: addv.8h h0, v0
-; CHECK-SD-NEXT: fmov w8, s0
-; CHECK-SD-NEXT: and w0, w8, #0xff
-; CHECK-SD-NEXT: add sp, sp, #16
+; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: convert_large_vector:
>From 803c102dcf3f37046897429232f291d8830782bb Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs at live.co.uk>
Date: Fri, 28 Aug 2026 11:37:18 +0100
Subject: [PATCH 2/2] move test
---
.../vec-combine-compare-to-bitmask-wide.ll | 339 ------------------
.../AArch64/vec-combine-compare-to-bitmask.ll | 336 +++++++++++++++++
2 files changed, 336 insertions(+), 339 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
deleted file mode 100644
index d968265b24136..0000000000000
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask-wide.ll
+++ /dev/null
@@ -1,339 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
-; RUN: llc -mtriple=aarch64_be -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
-
-; Bitmasks (bitcast <N x i1> to iN) from masks legalized out of >128-bit
-; sources. These all lower by narrowing the per-lane width until the mask fits
-; a 128-bit vector (the lane count, and so the bitmask, is unchanged), then
-; reusing the register-only compare->bitmask lowering. None of them may touch
-; the stack: the old lowering bailed here and fell into the generic
-; bitcast-through-memory legalization, leaving behind a dead frame.
-
-; 256-bit source, 4 lanes: narrowed <4 x i64> -> <4 x i32>.
-define i4 @bitmask_v4i64(<4 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v4i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI0_0
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv s0, v0.4s
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v4i64:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: adrp x8, .LCPI0_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
-; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
-; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT: addv s0, v0.4s
-; CHECK-BE-NEXT: fmov w0, s0
-; CHECK-BE-NEXT: ret
- %c = icmp ne <4 x i64> %v, zeroinitializer
- %b = bitcast <4 x i1> %c to i4
- ret i4 %b
-}
-
-; 256-bit source, 8 lanes: narrowed <8 x i32> -> <8 x i16>.
-define i8 @bitmask_v8i32(<8 x i32> %v) {
-; CHECK-LE-LABEL: bitmask_v8i32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI1_0
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v8i32:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI1_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w0, s0
-; CHECK-BE-NEXT: ret
- %c = icmp ne <8 x i32> %v, zeroinitializer
- %b = bitcast <8 x i1> %c to i8
- ret i8 %b
-}
-
-; 512-bit source, 8 lanes: narrowed <8 x i64> -> <8 x i16>.
-define i8 @bitmask_v8i64(<8 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v8i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI2_0
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v8i64:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: adrp x8, .LCPI2_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
-; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w0, s0
-; CHECK-BE-NEXT: ret
- %c = icmp ne <8 x i64> %v, zeroinitializer
- %b = bitcast <8 x i1> %c to i8
- ret i8 %b
-}
-
-; 512-bit source, 16 lanes: narrowed <16 x i32> -> <16 x i8>.
-define i16 @bitmask_v16i32(<16 x i32> %v) {
-; CHECK-LE-LABEL: bitmask_v16i32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v3.4s, v3.4s, #0
-; CHECK-LE-NEXT: cmeq v2.4s, v2.4s, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI3_0
-; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
-; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w0, v0.h[0]
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v16i32:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI3_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
-; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
-; CHECK-BE-NEXT: cmeq v3.4s, v3.4s, #0
-; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-BE-NEXT: cmeq v2.4s, v2.4s, #0
-; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-BE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w0, v0.h[0]
-; CHECK-BE-NEXT: ret
- %c = icmp ne <16 x i32> %v, zeroinitializer
- %b = bitcast <16 x i1> %c to i16
- ret i16 %b
-}
-
-; 1024-bit source, 16 lanes: narrowed <16 x i64> -> <16 x i8> (maximum
-; narrowing ratio: 8x).
-define i16 @bitmask_v16i64(<16 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v16i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v7.2d, v7.2d, #0
-; CHECK-LE-NEXT: cmeq v6.2d, v6.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI4_0
-; CHECK-LE-NEXT: cmeq v5.2d, v5.2d, #0
-; CHECK-LE-NEXT: cmeq v4.2d, v4.2d, #0
-; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-LE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-LE-NEXT: mvn v1.16b, v1.16b
-; CHECK-LE-NEXT: mvn v0.16b, v0.16b
-; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
-; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w0, v0.h[0]
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v16i64:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: ext v6.16b, v6.16b, v6.16b, #8
-; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #8
-; CHECK-BE-NEXT: adrp x8, .LCPI4_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI4_0
-; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
-; CHECK-BE-NEXT: ext v5.16b, v5.16b, v5.16b, #8
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
-; CHECK-BE-NEXT: cmeq v7.2d, v7.2d, #0
-; CHECK-BE-NEXT: cmeq v6.2d, v6.2d, #0
-; CHECK-BE-NEXT: cmeq v4.2d, v4.2d, #0
-; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-BE-NEXT: cmeq v5.2d, v5.2d, #0
-; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-BE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-BE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-BE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
-; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-BE-NEXT: mvn v1.16b, v1.16b
-; CHECK-BE-NEXT: mvn v0.16b, v0.16b
-; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w0, v0.h[0]
-; CHECK-BE-NEXT: ret
- %c = icmp ne <16 x i64> %v, zeroinitializer
- %b = bitcast <16 x i1> %c to i16
- ret i16 %b
-}
-
-; Float compares from a wide source take the same path.
-define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
-; CHECK-LE-LABEL: bitmask_v8f32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
-; CHECK-LE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
-; CHECK-LE-NEXT: adrp x8, .LCPI5_0
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v8f32:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI5_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI5_0
-; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
-; CHECK-BE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
-; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w0, s0
-; CHECK-BE-NEXT: ret
- %c = fcmp oeq <8 x float> %a, %b
- %m = bitcast <8 x i1> %c to i8
- ret i8 %m
-}
-
-; With SVE the wide compare stays in a predicate; the bitmask is still built
-; register-only (predicate -> Z register -> narrow -> weights+reduce).
-; TODO: a predicated UADDV over the weight vector would shorten this further.
-define i8 @bitmask_v8i32_sve(<8 x i32> %v) vscale_range(2,2) "target-features"="+sve" {
-; CHECK-LE-LABEL: bitmask_v8i32_sve:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: ptrue p0.s, vl4
-; CHECK-LE-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-LE-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-LE-NEXT: adrp x8, .LCPI6_0
-; CHECK-LE-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-LE-NEXT: ptrue p0.s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI6_0]
-; CHECK-LE-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; CHECK-LE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-LE-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
-;
-; CHECK-BE-LABEL: bitmask_v8i32_sve:
-; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI6_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
-; CHECK-BE-NEXT: ptrue p0.s, vl4
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-BE-NEXT: ptrue p0.s
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
-; CHECK-BE-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; CHECK-BE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-BE-NEXT: revb z0.s, p0/m, z0.s
-; CHECK-BE-NEXT: ptrue p0.h
-; CHECK-BE-NEXT: revb z0.h, p0/m, z0.h
-; CHECK-BE-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w0, s0
-; CHECK-BE-NEXT: ret
- %c = icmp ne <8 x i32> %v, zeroinitializer
- %b = bitcast <8 x i1> %c to i8
- ret i8 %b
-}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index 31429213cd2cd..7d20f5dd9d869 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -aarch64-enable-collect-loh=false -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -aarch64-enable-collect-loh=false -global-isel -global-isel-abort=2 -verify-machineinstrs < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -mtriple=aarch64_be -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-BE
; CHECK-GI: warning: Instruction selection used fallback path for convert_to_bitmask2
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for convert_to_bitmask_2xi32
@@ -1219,3 +1221,337 @@ define <32 x i1> @bitmask_v32i8_split(<32 x i8> %a, <32 x i8> %b) {
%r = icmp eq <32 x i8> %a, %b
ret <32 x i1> %r
}
+
+; Bitmasks (bitcast <N x i1> to iN) from masks legalized out of >128-bit
+; sources. These all lower by narrowing the per-lane width until the mask fits
+; a 128-bit vector (the lane count, and so the bitmask, is unchanged), then
+; reusing the register-only compare->bitmask lowering. None of them may touch
+; the stack: the old lowering bailed here and fell into the generic
+; bitcast-through-memory legalization, leaving behind a dead frame.
+
+; 256-bit source, 4 lanes: narrowed <4 x i64> -> <4 x i32>.
+define i4 @bitmask_v4i64(<4 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v4i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI22_0
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI22_0]
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv s0, v0.4s
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v4i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI22_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <4 x i64> %v, zeroinitializer
+ %b = bitcast <4 x i1> %c to i4
+ ret i4 %b
+}
+
+; 256-bit source, 8 lanes: narrowed <8 x i32> -> <8 x i16>.
+define i8 @bitmask_v8i32(<8 x i32> %v) {
+; CHECK-LE-LABEL: bitmask_v8i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI23_0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI23_0]
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI23_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI23_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i32> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
+
+; 512-bit source, 8 lanes: narrowed <8 x i64> -> <8 x i16>.
+define i8 @bitmask_v8i64(<8 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v8i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI24_0
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI24_0]
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI24_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI24_0
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i64> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
+
+; 512-bit source, 16 lanes: narrowed <16 x i32> -> <16 x i8>.
+define i16 @bitmask_v16i32(<16 x i32> %v) {
+; CHECK-LE-LABEL: bitmask_v16i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v3.4s, v3.4s, #0
+; CHECK-LE-NEXT: cmeq v2.4s, v2.4s, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI25_0
+; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI25_0]
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w0, v0.h[0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v16i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI25_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_0
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v3.4s, v3.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmeq v2.4s, v2.4s, #0
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <16 x i32> %v, zeroinitializer
+ %b = bitcast <16 x i1> %c to i16
+ ret i16 %b
+}
+
+; 1024-bit source, 16 lanes: narrowed <16 x i64> -> <16 x i8> (maximum
+; narrowing ratio: 8x).
+define i16 @bitmask_v16i64(<16 x i64> %v) {
+; CHECK-LE-LABEL: bitmask_v16i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmeq v7.2d, v7.2d, #0
+; CHECK-LE-NEXT: cmeq v6.2d, v6.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI26_0
+; CHECK-LE-NEXT: cmeq v5.2d, v5.2d, #0
+; CHECK-LE-NEXT: cmeq v4.2d, v4.2d, #0
+; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-LE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-LE-NEXT: mvn v1.16b, v1.16b
+; CHECK-LE-NEXT: mvn v0.16b, v0.16b
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI26_0]
+; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w0, v0.h[0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v16i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v6.16b, v6.16b, v6.16b, #8
+; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI26_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_0
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8
+; CHECK-BE-NEXT: ext v5.16b, v5.16b, v5.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmeq v7.2d, v7.2d, #0
+; CHECK-BE-NEXT: cmeq v6.2d, v6.2d, #0
+; CHECK-BE-NEXT: cmeq v4.2d, v4.2d, #0
+; CHECK-BE-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-BE-NEXT: cmeq v5.2d, v5.2d, #0
+; CHECK-BE-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-BE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-BE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BE-NEXT: mvn v1.16b, v1.16b
+; CHECK-BE-NEXT: mvn v0.16b, v0.16b
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <16 x i64> %v, zeroinitializer
+ %b = bitcast <16 x i1> %c to i16
+ ret i16 %b
+}
+
+; Float compares from a wide source take the same path.
+define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LE-LABEL: bitmask_v8f32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-LE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-LE-NEXT: adrp x8, .LCPI27_0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI27_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8f32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI27_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI27_0
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-BE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = fcmp oeq <8 x float> %a, %b
+ %m = bitcast <8 x i1> %c to i8
+ ret i8 %m
+}
+
+; With SVE the wide compare stays in a predicate; the bitmask is still built
+; register-only (predicate -> Z register -> narrow -> weights+reduce).
+; TODO: a predicated UADDV over the weight vector would shorten this further.
+define i8 @bitmask_v8i32_sve(<8 x i32> %v) vscale_range(2,2) "target-features"="+sve" {
+; CHECK-LE-LABEL: bitmask_v8i32_sve:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ptrue p0.s, vl4
+; CHECK-LE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-LE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-LE-NEXT: adrp x8, .LCPI28_0
+; CHECK-LE-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-LE-NEXT: ptrue p0.s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI28_0]
+; CHECK-LE-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-LE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-LE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w0, s0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v8i32_sve:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI28_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI28_0
+; CHECK-BE-NEXT: ptrue p0.s, vl4
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-BE-NEXT: ptrue p0.s
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-BE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-BE-NEXT: revb z0.s, p0/m, z0.s
+; CHECK-BE-NEXT: ptrue p0.h
+; CHECK-BE-NEXT: revb z0.h, p0/m, z0.h
+; CHECK-BE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+ %c = icmp ne <8 x i32> %v, zeroinitializer
+ %b = bitcast <8 x i1> %c to i8
+ ret i8 %b
+}
More information about the llvm-commits
mailing list