[llvm] [AArch64] Update and regenerate vec-combine-compare-to-bitmask.ll. NFC (PR #222242)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 22:48:15 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/222242
None
>From 7ef3a81eb2713a5b80a83887a599831160e84475 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 9 Sep 2026 06:48:04 +0100
Subject: [PATCH] [AArch64] Update and regenerate
vec-combine-compare-to-bitmask.ll. NFC
---
llvm/test/CodeGen/AArch64/arm64-vmul.ll | 2 +-
.../AArch64/vec-combine-compare-to-bitmask.ll | 1916 ++++++++++++-----
2 files changed, 1319 insertions(+), 599 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/arm64-vmul.ll b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
index 7bd5bb2c18ee3..ceb96f5cc04dd 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vmul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
@@ -2,7 +2,7 @@
; RUN: llc -mtriple=aarch64-none-elf -mattr=+aes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-elf -mattr=+aes -global-isel -global-isel-abort=2 2>&1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for fmls_2s
+; CHECK-GI: warning: Instruction selection used fallback path for fmls_2s
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_4s
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_2d
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_commuted_neg_2s
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index 57e3c9e2c93c6..be6905d06097e 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -aarch64-enable-collect-loh=false -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -aarch64-enable-collect-loh=false -global-isel -global-isel-abort=2 -verify-machineinstrs < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64_be -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-BE
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
; CHECK-GI: warning: Instruction selection used fallback path for convert_to_bitmask2
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for convert_to_bitmask_2xi32
@@ -16,63 +15,79 @@
define i16 @convert_to_bitmask16(<16 x i8> %vec) {
; Bits used in mask
; CHECK-SD-LABEL: convert_to_bitmask16:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: adrp x8, lCPI0_0 at PAGE
-; CHECK-SD-NEXT: cmeq.16b v0, v0, #0
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI0_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: umov.h w0, v0[0]
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: adrp x8, .LCPI0_0
+; CHECK-SD-NEXT: cmeq v0.16b, v0.16b, #0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: umov w0, v0.h[0]
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI0_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask16:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.16b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
+; CHECK-GI-NEXT: umov w10, v0.b[8]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
+; CHECK-GI-NEXT: umov w11, v0.b[9]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
+; CHECK-GI-NEXT: umov w10, v0.b[10]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
+; CHECK-GI-NEXT: umov w11, v0.b[11]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
+; CHECK-GI-NEXT: umov w10, v0.b[12]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
+; CHECK-GI-NEXT: umov w11, v0.b[13]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
+; CHECK-GI-NEXT: umov w10, v0.b[14]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
+; CHECK-GI-NEXT: umov w11, v0.b[15]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
@@ -92,39 +107,53 @@ define i16 @convert_to_bitmask16(<16 x i8> %vec) {
define i16 @convert_to_bitmask8(<8 x i16> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask8:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: adrp x8, lCPI1_0 at PAGE
-; CHECK-SD-NEXT: cmeq.8h v0, v0, #0
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI1_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: addv.8h h0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: adrp x8, .LCPI1_0
+; CHECK-SD-NEXT: cmeq v0.8h, v0.8h, #0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
; CHECK-SD-NEXT: fmov w8, s0
; CHECK-SD-NEXT: and w0, w8, #0xff
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: adrp x8, .LCPI1_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: and w0, w8, #0xff
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask8:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.8h v0, v0, v0
-; CHECK-GI-NEXT: xtn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: cmtst v0.8h, v0.8h, v0.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
@@ -144,24 +173,37 @@ define i16 @convert_to_bitmask8(<8 x i16> %vec) {
define i4 @convert_to_bitmask4(<4 x i32> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask4:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: adrp x8, lCPI2_0 at PAGE
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI2_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: adrp x8, .LCPI2_0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask4:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI2_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask4:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -181,15 +223,28 @@ define i4 @convert_to_bitmask4(<4 x i32> %vec) {
define i8 @convert_to_bitmask2(<2 x i64> %vec) {
; CHECK-LABEL: convert_to_bitmask2:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: adrp x8, lCPI3_0 at PAGE
-; CHECK-NEXT: cmeq.2d v0, v0, #0
-; CHECK-NEXT: ldr q1, [x8, lCPI3_0 at PAGEOFF]
-; CHECK-NEXT: bic.16b v0, v1, v0
-; CHECK-NEXT: addp.2d d0, v0
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: addp d0, v0.2d
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: and w0, w8, #0x3
; CHECK-NEXT: ret
+;
+; CHECK-BE-LABEL: convert_to_bitmask2:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI3_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
+; CHECK-BE-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addp d0, v0.2d
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: and w0, w8, #0x3
+; CHECK-BE-NEXT: ret
%cmp_result = icmp ne <2 x i64> %vec, zeroinitializer
@@ -201,39 +256,52 @@ define i8 @convert_to_bitmask2(<2 x i64> %vec) {
; Clang's __builtin_convertvector adds an undef vector concat for vectors with <8 elements.
define i8 @clang_builtins_undef_concat_convert_to_bitmask4(<4 x i32> %vec) {
; CHECK-SD-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: adrp x8, lCPI4_0 at PAGE
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI4_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: adrp x8, .LCPI4_0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI4_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI4_0
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: xtn.4h v0, v0
-; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
@@ -254,26 +322,43 @@ define i8 @clang_builtins_undef_concat_convert_to_bitmask4(<4 x i32> %vec) {
define i4 @convert_to_bitmask_no_compare(<4 x i32> %vec1, <4 x i32> %vec2) {
; CHECK-SD-LABEL: convert_to_bitmask_no_compare:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: adrp x8, lCPI5_0 at PAGE
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI5_0 at PAGEOFF]
-; CHECK-SD-NEXT: shl.4s v0, v0, #31
-; CHECK-SD-NEXT: cmlt.4s v0, v0, #0
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: adrp x8, .LCPI5_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-SD-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-SD-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_no_compare:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI5_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI5_0
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_no_compare:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: and.16b v0, v0, v1
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -294,28 +379,45 @@ define i4 @convert_to_bitmask_no_compare(<4 x i32> %vec1, <4 x i32> %vec2) {
define i4 @convert_to_bitmask_with_compare_chain(<4 x i32> %vec1, <4 x i32> %vec2) {
; CHECK-SD-LABEL: convert_to_bitmask_with_compare_chain:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmeq.4s v2, v0, #0
-; CHECK-SD-NEXT: cmeq.4s v0, v0, v1
-; CHECK-SD-NEXT: adrp x8, lCPI6_0 at PAGE
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI6_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v0, v2
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v2.4s, v0.4s, #0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: adrp x8, .LCPI6_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI6_0]
+; CHECK-SD-NEXT: bic v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_with_compare_chain:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI6_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v2.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_with_compare_chain:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmeq.4s v2, v0, #0
-; CHECK-GI-NEXT: cmeq.4s v0, v0, v1
-; CHECK-GI-NEXT: bic.16b v0, v0, v2
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: cmeq v2.4s, v0.4s, #0
+; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: bic v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -337,28 +439,46 @@ define i4 @convert_to_bitmask_with_compare_chain(<4 x i32> %vec1, <4 x i32> %vec
define i4 @convert_to_bitmask_with_trunc_in_chain(<4 x i32> %vec1, <4 x i32> %vec2) {
; CHECK-SD-LABEL: convert_to_bitmask_with_trunc_in_chain:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: adrp x8, lCPI7_0 at PAGE
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI7_0 at PAGEOFF]
-; CHECK-SD-NEXT: shl.4s v0, v0, #31
-; CHECK-SD-NEXT: cmlt.4s v0, v0, #0
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI7_0
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-SD-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-SD-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_with_trunc_in_chain:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI7_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI7_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_with_trunc_in_chain:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmeq.4s v0, v0, #0
-; CHECK-GI-NEXT: bic.16b v0, v1, v0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-GI-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -380,56 +500,86 @@ define i4 @convert_to_bitmask_with_trunc_in_chain(<4 x i32> %vec1, <4 x i32> %ve
define i4 @convert_to_bitmask_with_unknown_type_in_long_chain(<4 x i32> %vec1, <4 x i32> %vec2) {
; CHECK-SD-LABEL: convert_to_bitmask_with_unknown_type_in_long_chain:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: cmeq.4s v1, v1, #0
-; CHECK-SD-NEXT: adrp x8, lCPI8_0 at PAGE
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI8_0
; CHECK-SD-NEXT: movi d2, #0x000000ffffffff
; CHECK-SD-NEXT: movi d3, #0x00ffffffffffff
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
; CHECK-SD-NEXT: movi d1, #0xffff0000ffff0000
-; CHECK-SD-NEXT: xtn.4h v0, v0
-; CHECK-SD-NEXT: orr.8b v0, v0, v2
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v2.8b
; CHECK-SD-NEXT: movi d2, #0x00ffffffff0000
-; CHECK-SD-NEXT: eor.8b v1, v0, v1
-; CHECK-SD-NEXT: eor.8b v0, v0, v2
-; CHECK-SD-NEXT: mov.h v1[2], wzr
-; CHECK-SD-NEXT: orr.8b v0, v0, v3
-; CHECK-SD-NEXT: orr.8b v0, v1, v0
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI8_0 at PAGEOFF]
-; CHECK-SD-NEXT: shl.4h v0, v0, #15
-; CHECK-SD-NEXT: cmlt.4h v0, v0, #0
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: addv.4h h0, v0
+; CHECK-SD-NEXT: eor v1.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: eor v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT: mov v1.h[2], wzr
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v3.8b
+; CHECK-SD-NEXT: orr v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI8_0]
+; CHECK-SD-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-SD-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv h0, v0.4h
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_with_unknown_type_in_long_chain:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI8_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI8_0
+; CHECK-BE-NEXT: movi d2, #0x000000ffffffff
+; CHECK-BE-NEXT: movi d3, #0x00ffffffff0000
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: movi d1, #0xffff0000ffff0000
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: orr v0.8b, v0.8b, v2.8b
+; CHECK-BE-NEXT: movi d2, #0x00ffffffffffff
+; CHECK-BE-NEXT: eor v1.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: eor v0.8b, v0.8b, v3.8b
+; CHECK-BE-NEXT: mov v1.h[2], wzr
+; CHECK-BE-NEXT: orr v0.8b, v0.8b, v2.8b
+; CHECK-BE-NEXT: orr v0.8b, v1.8b, v0.8b
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_with_unknown_type_in_long_chain:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmeq.4s v0, v0, #0
-; CHECK-GI-NEXT: cmeq.4s v1, v1, #0
-; CHECK-GI-NEXT: adrp x8, lCPI8_4 at PAGE
-; CHECK-GI-NEXT: adrp x9, lCPI8_0 at PAGE
-; CHECK-GI-NEXT: ldr d2, [x9, lCPI8_0 at PAGEOFF]
-; CHECK-GI-NEXT: adrp x9, lCPI8_1 at PAGE
-; CHECK-GI-NEXT: bic.16b v0, v1, v0
-; CHECK-GI-NEXT: ldr d1, [x8, lCPI8_4 at PAGEOFF]
-; CHECK-GI-NEXT: adrp x8, lCPI8_3 at PAGE
-; CHECK-GI-NEXT: ldr d3, [x9, lCPI8_1 at PAGEOFF]
-; CHECK-GI-NEXT: xtn.4h v0, v0
-; CHECK-GI-NEXT: orr.8b v0, v0, v1
-; CHECK-GI-NEXT: ldr d1, [x8, lCPI8_3 at PAGEOFF]
-; CHECK-GI-NEXT: adrp x8, lCPI8_2 at PAGE
-; CHECK-GI-NEXT: eor.8b v1, v0, v1
-; CHECK-GI-NEXT: eor.8b v0, v2, v0
-; CHECK-GI-NEXT: ldr d2, [x8, lCPI8_2 at PAGEOFF]
-; CHECK-GI-NEXT: and.8b v1, v1, v2
-; CHECK-GI-NEXT: orr.8b v0, v3, v0
-; CHECK-GI-NEXT: orr.8b v0, v1, v0
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: mov.s w8, v0[3]
+; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-GI-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-GI-NEXT: adrp x8, .LCPI8_4
+; CHECK-GI-NEXT: adrp x9, .LCPI8_0
+; CHECK-GI-NEXT: ldr d2, [x9, :lo12:.LCPI8_0]
+; CHECK-GI-NEXT: adrp x9, .LCPI8_1
+; CHECK-GI-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI8_4]
+; CHECK-GI-NEXT: adrp x8, .LCPI8_3
+; CHECK-GI-NEXT: ldr d3, [x9, :lo12:.LCPI8_1]
+; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: orr v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI8_3]
+; CHECK-GI-NEXT: adrp x8, .LCPI8_2
+; CHECK-GI-NEXT: eor v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT: eor v0.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT: ldr d2, [x8, :lo12:.LCPI8_2]
+; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT: orr v0.8b, v3.8b, v0.8b
+; CHECK-GI-NEXT: orr v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: mov w8, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: lsl w8, w8, #3
; CHECK-GI-NEXT: orr w8, w8, #0x7
@@ -456,31 +606,48 @@ define i4 @convert_to_bitmask_with_unknown_type_in_long_chain(<4 x i32> %vec1, <
define i4 @convert_to_bitmask_with_different_types_in_chain(<4 x i16> %vec1, <4 x i32> %vec2) {
; CHECK-SD-LABEL: convert_to_bitmask_with_different_types_in_chain:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmeq.4s v1, v1, #0
-; CHECK-SD-NEXT: cmeq.4h v0, v0, #0
-; CHECK-SD-NEXT: adrp x8, lCPI9_0 at PAGE
-; CHECK-SD-NEXT: xtn.4h v1, v1
-; CHECK-SD-NEXT: orn.8b v0, v1, v0
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI9_0 at PAGEOFF]
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: addv.4h h0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmeq v0.4h, v0.4h, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI9_0
+; CHECK-SD-NEXT: xtn v1.4h, v1.4s
+; CHECK-SD-NEXT: orn v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI9_0]
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv h0, v0.4h
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_with_different_types_in_chain:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: adrp x8, .LCPI9_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI9_0
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmeq v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: xtn v1.4h, v1.4s
+; CHECK-BE-NEXT: orn v0.8b, v1.8b, v0.8b
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_with_different_types_in_chain:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmeq.4s v1, v1, #0
-; CHECK-GI-NEXT: cmeq.4h v0, v0, #0
-; CHECK-GI-NEXT: xtn.4h v1, v1
-; CHECK-GI-NEXT: orn.8b v0, v1, v0
-; CHECK-GI-NEXT: ushll.4s v0, v0, #0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-GI-NEXT: cmeq v0.4h, v0.4h, #0
+; CHECK-GI-NEXT: xtn v1.4h, v1.4s
+; CHECK-GI-NEXT: orn v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -502,63 +669,80 @@ define i4 @convert_to_bitmask_with_different_types_in_chain(<4 x i16> %vec1, <4
define i16 @convert_to_bitmask_without_knowing_type(<16 x i1> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask_without_knowing_type:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: shl.16b v0, v0, #7
-; CHECK-SD-NEXT: adrp x8, lCPI10_0 at PAGE
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI10_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmlt.16b v0, v0, #0
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: umov.h w0, v0[0]
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-SD-NEXT: adrp x8, .LCPI10_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
+; CHECK-SD-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: umov w0, v0.h[0]
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_without_knowing_type:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI10_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI10_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w0, v0.h[0]
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_without_knowing_type:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
+; CHECK-GI-NEXT: umov w10, v0.b[8]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
+; CHECK-GI-NEXT: umov w11, v0.b[9]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
+; CHECK-GI-NEXT: umov w10, v0.b[10]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
+; CHECK-GI-NEXT: umov w11, v0.b[11]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
+; CHECK-GI-NEXT: umov w10, v0.b[12]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
+; CHECK-GI-NEXT: umov w11, v0.b[13]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
+; CHECK-GI-NEXT: umov w10, v0.b[14]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
+; CHECK-GI-NEXT: umov w11, v0.b[15]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
@@ -575,15 +759,27 @@ define i16 @convert_to_bitmask_without_knowing_type(<16 x i1> %vec) {
define i2 @convert_to_bitmask_2xi32(<2 x i32> %vec) {
; CHECK-LABEL: convert_to_bitmask_2xi32:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: mov x8, #1 ; =0x1
-; CHECK-NEXT: cmeq.2s v0, v0, #0
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov x8, #1 // =0x1
+; CHECK-NEXT: cmeq v0.2s, v0.2s, #0
; CHECK-NEXT: movk x8, #2, lsl #32
; CHECK-NEXT: fmov d1, x8
-; CHECK-NEXT: bic.8b v0, v1, v0
-; CHECK-NEXT: addp.2s v0, v0, v0
+; CHECK-NEXT: bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
+;
+; CHECK-BE-LABEL: convert_to_bitmask_2xi32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: adrp x8, .LCPI11_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_0
+; CHECK-BE-NEXT: ld1 { v1.2s }, [x8]
+; CHECK-BE-NEXT: cmeq v0.2s, v0.2s, #0
+; CHECK-BE-NEXT: bic v0.8b, v1.8b, v0.8b
+; CHECK-BE-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
%cmp_result = icmp ne <2 x i32> %vec, zeroinitializer
%bitmask = bitcast <2 x i1> %cmp_result to i2
@@ -592,28 +788,41 @@ define i2 @convert_to_bitmask_2xi32(<2 x i32> %vec) {
define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask_4xi8:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: bic.4h v0, #255, lsl #8
-; CHECK-SD-NEXT: adrp x8, lCPI12_0 at PAGE
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI12_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmeq.4h v0, v0, #0
-; CHECK-SD-NEXT: bic.8b v0, v1, v0
-; CHECK-SD-NEXT: addv.4h h0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT: adrp x8, .LCPI12_0
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI12_0]
+; CHECK-SD-NEXT: cmeq v0.4h, v0.4h, #0
+; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT: addv h0, v0.4h
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_4xi8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: adrp x8, .LCPI12_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_0
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-BE-NEXT: cmeq v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: bic v0.8b, v1.8b, v0.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_4xi8:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
-; CHECK-GI-NEXT: cmeq.8b v0, v0, v1
-; CHECK-GI-NEXT: mvn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: cmeq v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT: mvn v0.8b, v0.8b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
@@ -632,40 +841,53 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
define i8 @convert_to_bitmask_8xi2(<8 x i2> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask_8xi2:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: movi.8b v1, #3
-; CHECK-SD-NEXT: adrp x8, lCPI13_0 at PAGE
-; CHECK-SD-NEXT: cmtst.8b v0, v0, v1
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: addv.8b b0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: movi v1.8b, #3
+; CHECK-SD-NEXT: adrp x8, .LCPI13_0
+; CHECK-SD-NEXT: cmtst v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI13_0]
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv b0, v0.8b
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_8xi2:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: movi v1.8b, #3
+; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-NEXT: adrp x8, .LCPI13_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI13_0
+; CHECK-BE-NEXT: cmtst v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: ld1 { v1.8b }, [x8]
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv b0, v0.8b
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_8xi2:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: movi.8b v1, #3
-; CHECK-GI-NEXT: and.8b v0, v0, v1
-; CHECK-GI-NEXT: cmtst.8b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: movi v1.8b, #3
+; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT: cmtst v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
@@ -683,28 +905,43 @@ define i8 @convert_to_bitmask_8xi2(<8 x i2> %vec) {
define i4 @convert_to_bitmask_float(<4 x float> %vec) {
; CHECK-SD-LABEL: convert_to_bitmask_float:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: fcmgt.4s v1, v0, #0.0
-; CHECK-SD-NEXT: fcmlt.4s v0, v0, #0.0
-; CHECK-SD-NEXT: adrp x8, lCPI14_0 at PAGE
-; CHECK-SD-NEXT: orr.16b v0, v0, v1
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI14_0 at PAGEOFF]
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: fcmgt v1.4s, v0.4s, #0.0
+; CHECK-SD-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-SD-NEXT: adrp x8, .LCPI14_0
+; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_to_bitmask_float:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI14_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI14_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: fcmgt v1.4s, v0.4s, #0.0
+; CHECK-BE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-BE-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_to_bitmask_float:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: fcmgt.4s v1, v0, #0.0
-; CHECK-GI-NEXT: fcmlt.4s v0, v0, #0.0
-; CHECK-GI-NEXT: orr.16b v0, v0, v1
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: fcmgt v1.4s, v0.4s, #0.0
+; CHECK-GI-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -726,42 +963,59 @@ define i4 @convert_to_bitmask_float(<4 x float> %vec) {
; After the comparison against 0, this is truncated to <8 x i16>, which is valid again.
define i8 @convert_large_vector(<8 x i32> %vec) {
; CHECK-SD-LABEL: convert_large_vector:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmeq.4s v1, v1, #0
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: adrp x8, lCPI15_0 at PAGE
-; CHECK-SD-NEXT: uzp1.8h v0, v0, v1
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI15_0 at PAGEOFF]
-; CHECK-SD-NEXT: bic.16b v0, v1, v0
-; CHECK-SD-NEXT: addv.8h h0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI15_0
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI15_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_large_vector:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI15_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI15_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_large_vector:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: cmtst.4s v1, v1, v1
-; CHECK-GI-NEXT: uzp1.8h v0, v0, v1
-; CHECK-GI-NEXT: xtn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
@@ -780,28 +1034,43 @@ define i8 @convert_large_vector(<8 x i32> %vec) {
define i4 @convert_legalized_illegal_element_size(<4 x i22> %vec) {
; CHECK-SD-LABEL: convert_legalized_illegal_element_size:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: movi.4s v1, #63, msl #16
-; CHECK-SD-NEXT: adrp x8, lCPI16_0 at PAGE
-; CHECK-SD-NEXT: cmtst.4s v0, v0, v1
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI16_0 at PAGEOFF]
-; CHECK-SD-NEXT: xtn.4h v0, v0
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: addv.4h h0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: movi v1.4s, #63, msl #16
+; CHECK-SD-NEXT: adrp x8, .LCPI16_0
+; CHECK-SD-NEXT: cmtst v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI16_0]
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv h0, v0.4h
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: convert_legalized_illegal_element_size:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: movi v1.4s, #63, msl #16
+; CHECK-BE-NEXT: adrp x8, .LCPI16_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI16_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmtst v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: convert_legalized_illegal_element_size:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: movi.4s v1, #63, msl #16
-; CHECK-GI-NEXT: and.16b v0, v0, v1
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: movi v1.4s, #63, msl #16
+; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov w10, v0.s[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
@@ -821,51 +1090,75 @@ define i4 @convert_legalized_illegal_element_size(<4 x i22> %vec) {
; This may still be converted as a v8i8 after the vector concat (but not as v4iX).
define i8 @no_direct_convert_for_bad_concat(<4 x i32> %vec) {
; CHECK-SD-LABEL: no_direct_convert_for_bad_concat:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: cmtst.4s v0, v0, v0
-; CHECK-SD-NEXT: adrp x8, lCPI17_0 at PAGE
-; CHECK-SD-NEXT: xtn.4h v0, v0
-; CHECK-SD-NEXT: umov.h w9, v0[0]
-; CHECK-SD-NEXT: mov.b v1[4], w9
-; CHECK-SD-NEXT: umov.h w9, v0[1]
-; CHECK-SD-NEXT: mov.b v1[5], w9
-; CHECK-SD-NEXT: umov.h w9, v0[2]
-; CHECK-SD-NEXT: mov.b v1[6], w9
-; CHECK-SD-NEXT: umov.h w9, v0[3]
-; CHECK-SD-NEXT: mov.b v1[7], w9
-; CHECK-SD-NEXT: shl.8b v0, v1, #7
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI17_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmlt.8b v0, v0, #0
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: addv.8b b0, v0
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: adrp x8, .LCPI17_0
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: umov w9, v0.h[0]
+; CHECK-SD-NEXT: mov v1.b[4], w9
+; CHECK-SD-NEXT: umov w9, v0.h[1]
+; CHECK-SD-NEXT: mov v1.b[5], w9
+; CHECK-SD-NEXT: umov w9, v0.h[2]
+; CHECK-SD-NEXT: mov v1.b[6], w9
+; CHECK-SD-NEXT: umov w9, v0.h[3]
+; CHECK-SD-NEXT: mov v1.b[7], w9
+; CHECK-SD-NEXT: shl v0.8b, v1.8b, #7
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI17_0]
+; CHECK-SD-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv b0, v0.8b
; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: no_direct_convert_for_bad_concat:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI17_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI17_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: umov w9, v0.h[0]
+; CHECK-BE-NEXT: mov v1.b[4], w9
+; CHECK-BE-NEXT: umov w9, v0.h[1]
+; CHECK-BE-NEXT: mov v1.b[5], w9
+; CHECK-BE-NEXT: umov w9, v0.h[2]
+; CHECK-BE-NEXT: mov v1.b[6], w9
+; CHECK-BE-NEXT: umov w9, v0.h[3]
+; CHECK-BE-NEXT: mov v1.b[7], w9
+; CHECK-BE-NEXT: shl v0.8b, v1.8b, #7
+; CHECK-BE-NEXT: ld1 { v1.8b }, [x8]
+; CHECK-BE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv b0, v0.8b
+; CHECK-BE-NEXT: fmov w0, s0
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: no_direct_convert_for_bad_concat:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: xtn.4h v0, v0
-; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
-; CHECK-GI-NEXT: mov.s v0[1], v0[0]
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT: mov v0.s[1], v0.s[0]
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: umov w11, v0.b[5]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: umov w10, v0.b[6]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
; CHECK-GI-NEXT: and w9, w10, #0x1
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
@@ -884,10 +1177,19 @@ define i8 @no_direct_convert_for_bad_concat(<4 x i32> %vec) {
define <8 x i1> @no_convert_without_direct_bitcast(<8 x i16> %vec) {
; CHECK-LABEL: no_convert_without_direct_bitcast:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: cmtst.8h v0, v0, v0
-; CHECK-NEXT: xtn.8b v0, v0
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmtst v0.8h, v0.8h, v0.8h
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: ret
+;
+; CHECK-BE-LABEL: no_convert_without_direct_bitcast:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmtst v0.8h, v0.8h, v0.8h
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-NEXT: ret
%cmp_result = icmp ne <8 x i16> %vec, zeroinitializer
ret <8 x i1> %cmp_result
@@ -895,59 +1197,89 @@ define <8 x i1> @no_convert_without_direct_bitcast(<8 x i16> %vec) {
define i6 @no_combine_illegal_num_elements(<6 x i32> %vec) {
; CHECK-SD-LABEL: no_combine_illegal_num_elements:
-; CHECK-SD: ; %bb.0:
+; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: fmov s1, w4
-; CHECK-SD-NEXT: mov.s v0[1], w1
-; CHECK-SD-NEXT: mov.s v1[1], w5
-; CHECK-SD-NEXT: mov.s v0[2], w2
-; CHECK-SD-NEXT: cmeq.4s v1, v1, #0
-; CHECK-SD-NEXT: mov.s v0[3], w3
-; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
-; CHECK-SD-NEXT: uzp1.8h v0, v0, v1
-; CHECK-SD-NEXT: mvn.16b v0, v0
-; CHECK-SD-NEXT: xtn.8b v0, v0
-; CHECK-SD-NEXT: umov.b w8, v0[0]
-; CHECK-SD-NEXT: umov.b w9, v0[1]
-; CHECK-SD-NEXT: umov.b w10, v0[2]
+; CHECK-SD-NEXT: mov v0.s[1], w1
+; CHECK-SD-NEXT: mov v1.s[1], w5
+; CHECK-SD-NEXT: mov v0.s[2], w2
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: mov v0.s[3], w3
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: mvn v0.16b, v0.16b
+; CHECK-SD-NEXT: xtn v0.8b, v0.8h
+; CHECK-SD-NEXT: umov w8, v0.b[0]
+; CHECK-SD-NEXT: umov w9, v0.b[1]
+; CHECK-SD-NEXT: umov w10, v0.b[2]
; CHECK-SD-NEXT: and w8, w8, #0x1
; CHECK-SD-NEXT: bfi w8, w9, #1, #1
-; CHECK-SD-NEXT: umov.b w9, v0[3]
+; CHECK-SD-NEXT: umov w9, v0.b[3]
; CHECK-SD-NEXT: bfi w8, w10, #2, #1
-; CHECK-SD-NEXT: umov.b w10, v0[4]
+; CHECK-SD-NEXT: umov w10, v0.b[4]
; CHECK-SD-NEXT: bfi w8, w9, #3, #1
-; CHECK-SD-NEXT: umov.b w9, v0[5]
+; CHECK-SD-NEXT: umov w9, v0.b[5]
; CHECK-SD-NEXT: bfi w8, w10, #4, #1
; CHECK-SD-NEXT: orr w8, w8, w9, lsl #5
; CHECK-SD-NEXT: and w0, w8, #0x3f
; CHECK-SD-NEXT: add sp, sp, #16
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: no_combine_illegal_num_elements:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: fmov s0, w0
+; CHECK-BE-NEXT: fmov s1, w4
+; CHECK-BE-NEXT: mov v0.s[1], w1
+; CHECK-BE-NEXT: mov v1.s[1], w5
+; CHECK-BE-NEXT: mov v0.s[2], w2
+; CHECK-BE-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: mov v0.s[3], w3
+; CHECK-BE-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: mvn v0.16b, v0.16b
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: umov w8, v0.b[1]
+; CHECK-BE-NEXT: umov w9, v0.b[0]
+; CHECK-BE-NEXT: ubfiz w0, w8, #4, #1
+; CHECK-BE-NEXT: umov w8, v0.b[2]
+; CHECK-BE-NEXT: bfi w0, w9, #5, #1
+; CHECK-BE-NEXT: umov w9, v0.b[3]
+; CHECK-BE-NEXT: bfi w0, w8, #3, #1
+; CHECK-BE-NEXT: umov w8, v0.b[4]
+; CHECK-BE-NEXT: bfi w0, w9, #2, #1
+; CHECK-BE-NEXT: umov w9, v0.b[5]
+; CHECK-BE-NEXT: bfi w0, w8, #1, #1
+; CHECK-BE-NEXT: bfxil w0, w9, #0, #1
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: no_combine_illegal_num_elements:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: fmov s0, w0
; CHECK-GI-NEXT: fmov s1, w4
-; CHECK-GI-NEXT: movi.2d v2, #0000000000000000
-; CHECK-GI-NEXT: mov.s v0[1], w1
-; CHECK-GI-NEXT: mov.s v1[1], w5
-; CHECK-GI-NEXT: mov.s v0[2], w2
-; CHECK-GI-NEXT: cmeq.4s v1, v1, v2
-; CHECK-GI-NEXT: mov.s v0[3], w3
-; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
-; CHECK-GI-NEXT: mov.s w8, v0[1]
-; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT: mov v0.s[1], w1
+; CHECK-GI-NEXT: mov v1.s[1], w5
+; CHECK-GI-NEXT: mov v0.s[2], w2
+; CHECK-GI-NEXT: cmeq v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: mov v0.s[3], w3
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
; CHECK-GI-NEXT: fmov w11, s0
-; CHECK-GI-NEXT: mov.s w10, v0[3]
-; CHECK-GI-NEXT: mvn.16b v0, v1
+; CHECK-GI-NEXT: mov w10, v0.s[3]
+; CHECK-GI-NEXT: mvn v0.16b, v1.16b
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: bfi w11, w8, #1, #31
; CHECK-GI-NEXT: and w8, w9, #0x1
; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: mov.s w10, v0[1]
+; CHECK-GI-NEXT: mov w10, v0.s[1]
; CHECK-GI-NEXT: orr w8, w11, w8, lsl #2
; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
; CHECK-GI-NEXT: fmov w9, s0
@@ -969,210 +1301,232 @@ define i6 @no_combine_illegal_num_elements(<6 x i32> %vec) {
; Only apply the combine when casting a vector to a scalar.
define <2 x i8> @vector_to_vector_cast(<16 x i1> %arg) nounwind {
; CHECK-SD-LABEL: vector_to_vector_cast:
-; CHECK-SD: ; %bb.0:
+; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: sub sp, sp, #16
-; CHECK-SD-NEXT: shl.16b v0, v0, #7
-; CHECK-SD-NEXT: adrp x8, lCPI20_0 at PAGE
-; CHECK-SD-NEXT: ldr q1, [x8, lCPI20_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmlt.16b v0, v0, #0
-; CHECK-SD-NEXT: and.16b v0, v0, v1
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: addp.16b v0, v0, v0
-; CHECK-SD-NEXT: ushll.8h v0, v0, #0
-; CHECK-SD-NEXT: ushll.4s v0, v0, #0
-; CHECK-SD-NEXT: ; kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-SD-NEXT: adrp x8, .LCPI20_0
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI20_0]
+; CHECK-SD-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT: add sp, sp, #16
; CHECK-SD-NEXT: ret
;
+; CHECK-BE-LABEL: vector_to_vector_cast:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI20_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.8b, v0.8b
+; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+;
; CHECK-GI-LABEL: vector_to_vector_cast:
-; CHECK-GI: ; %bb.0:
+; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: mov d1, v0[1]
-; CHECK-GI-NEXT: umov.b w10, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w13, v0[0]
-; CHECK-GI-NEXT: umov.b w14, v0[2]
-; CHECK-GI-NEXT: umov.b w15, v0[3]
-; CHECK-GI-NEXT: umov.b w11, v0[2]
-; CHECK-GI-NEXT: umov.b w16, v0[4]
-; CHECK-GI-NEXT: umov.b w17, v0[5]
-; CHECK-GI-NEXT: umov.b w12, v0[3]
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: mov d1, v0.d[1]
+; CHECK-GI-NEXT: umov w10, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w13, v0.b[0]
+; CHECK-GI-NEXT: umov w14, v0.b[2]
+; CHECK-GI-NEXT: umov w15, v0.b[3]
+; CHECK-GI-NEXT: umov w11, v0.b[2]
+; CHECK-GI-NEXT: umov w16, v0.b[4]
+; CHECK-GI-NEXT: umov w17, v0.b[5]
+; CHECK-GI-NEXT: umov w12, v0.b[3]
; CHECK-GI-NEXT: and w8, w8, #0x1
; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w0, v1[1]
+; CHECK-GI-NEXT: umov w18, v1.b[1]
; CHECK-GI-NEXT: bfi w9, w8, #1, #31
; CHECK-GI-NEXT: bfi w13, w10, #1, #31
; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: umov.b w8, v1[0]
-; CHECK-GI-NEXT: umov.b w10, v1[2]
+; CHECK-GI-NEXT: umov w8, v1.b[0]
+; CHECK-GI-NEXT: umov w10, v1.b[2]
; CHECK-GI-NEXT: and w15, w15, #0x1
; CHECK-GI-NEXT: orr w13, w13, w14, lsl #2
-; CHECK-GI-NEXT: umov.b w14, v1[3]
+; CHECK-GI-NEXT: umov w14, v1.b[3]
; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: and w0, w0, #0x1
+; CHECK-GI-NEXT: and w18, w18, #0x1
; CHECK-GI-NEXT: and w16, w16, #0x1
; CHECK-GI-NEXT: orr w9, w9, w11, lsl #2
; CHECK-GI-NEXT: orr w13, w13, w15, lsl #3
-; CHECK-GI-NEXT: umov.b w15, v1[4]
-; CHECK-GI-NEXT: umov.b w11, v0[6]
-; CHECK-GI-NEXT: bfi w8, w0, #1, #31
+; CHECK-GI-NEXT: umov w15, v1.b[4]
+; CHECK-GI-NEXT: umov w11, v0.b[6]
+; CHECK-GI-NEXT: bfi w8, w18, #1, #31
; CHECK-GI-NEXT: and w10, w10, #0x1
; CHECK-GI-NEXT: and w17, w17, #0x1
; CHECK-GI-NEXT: orr w13, w13, w16, lsl #4
; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[7]
+; CHECK-GI-NEXT: umov w18, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w10, lsl #2
-; CHECK-GI-NEXT: umov.b w10, v1[5]
-; CHECK-GI-NEXT: umov.b w16, v1[6]
+; CHECK-GI-NEXT: umov w10, v1.b[5]
+; CHECK-GI-NEXT: umov w16, v1.b[6]
; CHECK-GI-NEXT: orr w13, w13, w17, lsl #5
-; CHECK-GI-NEXT: umov.b w17, v0[4]
+; CHECK-GI-NEXT: umov w17, v0.b[4]
; CHECK-GI-NEXT: and w15, w15, #0x1
; CHECK-GI-NEXT: orr w8, w8, w14, lsl #3
; CHECK-GI-NEXT: and w12, w12, #0x1
; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: umov.b w14, v1[7]
+; CHECK-GI-NEXT: umov w14, v1.b[7]
; CHECK-GI-NEXT: orr w9, w9, w12, lsl #3
; CHECK-GI-NEXT: orr w11, w13, w11, lsl #6
; CHECK-GI-NEXT: orr w8, w8, w15, lsl #4
-; CHECK-GI-NEXT: umov.b w15, v0[5]
+; CHECK-GI-NEXT: umov w15, v0.b[5]
; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: and w0, w0, #0x1
+; CHECK-GI-NEXT: and w18, w18, #0x1
; CHECK-GI-NEXT: and w12, w17, #0x1
-; CHECK-GI-NEXT: umov.b w13, v0[1]
+; CHECK-GI-NEXT: umov w13, v0.b[1]
; CHECK-GI-NEXT: orr w8, w8, w10, lsl #5
; CHECK-GI-NEXT: and w16, w16, #0x1
; CHECK-GI-NEXT: orr w9, w9, w12, lsl #4
-; CHECK-GI-NEXT: umov.b w10, v0[0]
-; CHECK-GI-NEXT: orr w11, w11, w0, lsl #7
+; CHECK-GI-NEXT: umov w10, v0.b[0]
+; CHECK-GI-NEXT: orr w11, w11, w18, lsl #7
; CHECK-GI-NEXT: and w14, w14, #0x1
; CHECK-GI-NEXT: and w12, w15, #0x1
-; CHECK-GI-NEXT: umov.b w15, v0[2]
+; CHECK-GI-NEXT: umov w15, v0.b[2]
; CHECK-GI-NEXT: orr w8, w8, w16, lsl #6
; CHECK-GI-NEXT: orr w9, w9, w12, lsl #5
-; CHECK-GI-NEXT: umov.b w12, v0[6]
+; CHECK-GI-NEXT: umov w12, v0.b[6]
; CHECK-GI-NEXT: strb w11, [sp, #8]
; CHECK-GI-NEXT: and w11, w13, #0x1
-; CHECK-GI-NEXT: umov.b w13, v0[3]
+; CHECK-GI-NEXT: umov w13, v0.b[3]
; CHECK-GI-NEXT: orr w8, w8, w14, lsl #7
-; CHECK-GI-NEXT: umov.b w14, v0[7]
+; CHECK-GI-NEXT: umov w14, v0.b[7]
; CHECK-GI-NEXT: ldr b0, [sp, #8]
; CHECK-GI-NEXT: bfi w10, w11, #1, #31
; CHECK-GI-NEXT: and w11, w15, #0x1
; CHECK-GI-NEXT: strb w8, [sp, #9]
-; CHECK-GI-NEXT: umov.b w15, v0[4]
+; CHECK-GI-NEXT: umov w15, v0.b[4]
; CHECK-GI-NEXT: and w8, w12, #0x1
; CHECK-GI-NEXT: orr w10, w10, w11, lsl #2
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #6
; CHECK-GI-NEXT: and w9, w13, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[1]
+; CHECK-GI-NEXT: umov w11, v0.b[1]
; CHECK-GI-NEXT: orr w9, w10, w9, lsl #3
-; CHECK-GI-NEXT: umov.b w10, v0[5]
-; CHECK-GI-NEXT: umov.b w12, v0[0]
+; CHECK-GI-NEXT: umov w10, v0.b[5]
+; CHECK-GI-NEXT: umov w12, v0.b[0]
; CHECK-GI-NEXT: and w13, w14, #0x1
-; CHECK-GI-NEXT: umov.b w16, v0[2]
-; CHECK-GI-NEXT: umov.b w17, v0[3]
+; CHECK-GI-NEXT: umov w16, v0.b[2]
+; CHECK-GI-NEXT: umov w17, v0.b[3]
; CHECK-GI-NEXT: and w14, w15, #0x1
-; CHECK-GI-NEXT: umov.b w15, v0[2]
+; CHECK-GI-NEXT: umov w15, v0.b[2]
; CHECK-GI-NEXT: orr w8, w8, w13, lsl #7
; CHECK-GI-NEXT: orr w9, w9, w14, lsl #4
-; CHECK-GI-NEXT: umov.b w13, v0[6]
+; CHECK-GI-NEXT: umov w13, v0.b[6]
; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[3]
+; CHECK-GI-NEXT: umov w14, v0.b[3]
; CHECK-GI-NEXT: strb w8, [sp, #10]
; CHECK-GI-NEXT: and w8, w10, #0x1
; CHECK-GI-NEXT: bfi w12, w11, #1, #31
; CHECK-GI-NEXT: orr w8, w9, w8, lsl #5
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: and w9, w15, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: umov.b w15, v0[1]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: umov w15, v0.b[1]
; CHECK-GI-NEXT: orr w9, w12, w9, lsl #2
-; CHECK-GI-NEXT: umov.b w12, v0[5]
+; CHECK-GI-NEXT: umov w12, v0.b[5]
; CHECK-GI-NEXT: and w13, w13, #0x1
; CHECK-GI-NEXT: and w14, w14, #0x1
; CHECK-GI-NEXT: orr w8, w8, w13, lsl #6
-; CHECK-GI-NEXT: umov.b w13, v0[0]
+; CHECK-GI-NEXT: umov w13, v0.b[0]
; CHECK-GI-NEXT: orr w9, w9, w14, lsl #3
; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[6]
+; CHECK-GI-NEXT: umov w14, v0.b[6]
; CHECK-GI-NEXT: and w11, w11, #0x1
; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[3]
+; CHECK-GI-NEXT: umov w18, v0.b[3]
; CHECK-GI-NEXT: orr w9, w9, w10, lsl #4
; CHECK-GI-NEXT: and w10, w12, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[7]
+; CHECK-GI-NEXT: umov w12, v0.b[7]
; CHECK-GI-NEXT: orr w8, w8, w11, lsl #7
; CHECK-GI-NEXT: bfi w13, w15, #1, #31
; CHECK-GI-NEXT: and w11, w16, #0x1
; CHECK-GI-NEXT: orr w9, w9, w10, lsl #5
; CHECK-GI-NEXT: and w10, w14, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[4]
+; CHECK-GI-NEXT: umov w14, v0.b[4]
; CHECK-GI-NEXT: strb w8, [sp, #11]
-; CHECK-GI-NEXT: umov.b w15, v0[1]
-; CHECK-GI-NEXT: umov.b w16, v0[3]
+; CHECK-GI-NEXT: umov w15, v0.b[1]
+; CHECK-GI-NEXT: umov w16, v0.b[3]
; CHECK-GI-NEXT: orr w8, w9, w10, lsl #6
; CHECK-GI-NEXT: orr w9, w13, w11, lsl #2
; CHECK-GI-NEXT: and w10, w12, #0x1
; CHECK-GI-NEXT: and w11, w17, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[5]
-; CHECK-GI-NEXT: umov.b w17, v0[0]
+; CHECK-GI-NEXT: umov w12, v0.b[5]
+; CHECK-GI-NEXT: umov w17, v0.b[0]
; CHECK-GI-NEXT: orr w8, w8, w10, lsl #7
; CHECK-GI-NEXT: orr w9, w9, w11, lsl #3
-; CHECK-GI-NEXT: umov.b w10, v0[1]
+; CHECK-GI-NEXT: umov w10, v0.b[1]
; CHECK-GI-NEXT: and w11, w14, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[0]
+; CHECK-GI-NEXT: umov w14, v0.b[0]
; CHECK-GI-NEXT: and w15, w15, #0x1
; CHECK-GI-NEXT: orr w9, w9, w11, lsl #4
-; CHECK-GI-NEXT: umov.b w11, v0[2]
-; CHECK-GI-NEXT: umov.b w13, v0[6]
+; CHECK-GI-NEXT: umov w11, v0.b[2]
+; CHECK-GI-NEXT: umov w13, v0.b[6]
; CHECK-GI-NEXT: and w12, w12, #0x1
; CHECK-GI-NEXT: bfi w17, w15, #1, #31
-; CHECK-GI-NEXT: umov.b w15, v0[5]
+; CHECK-GI-NEXT: umov w15, v0.b[5]
; CHECK-GI-NEXT: orr w9, w9, w12, lsl #5
; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[2]
+; CHECK-GI-NEXT: umov w12, v0.b[2]
; CHECK-GI-NEXT: bfi w14, w10, #1, #31
-; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: umov w10, v0.b[4]
; CHECK-GI-NEXT: ldr b1, [sp, #9]
; CHECK-GI-NEXT: and w11, w11, #0x1
; CHECK-GI-NEXT: and w13, w13, #0x1
; CHECK-GI-NEXT: strb w8, [sp, #12]
; CHECK-GI-NEXT: orr w11, w14, w11, lsl #2
; CHECK-GI-NEXT: and w14, w16, #0x1
-; CHECK-GI-NEXT: umov.b w16, v0[4]
+; CHECK-GI-NEXT: umov w16, v0.b[4]
; CHECK-GI-NEXT: and w12, w12, #0x1
; CHECK-GI-NEXT: and w15, w15, #0x1
; CHECK-GI-NEXT: orr w9, w9, w13, lsl #6
; CHECK-GI-NEXT: orr w11, w11, w14, lsl #3
; CHECK-GI-NEXT: orr w12, w17, w12, lsl #2
; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: and w17, w0, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[5]
-; CHECK-GI-NEXT: umov.b w14, v0[6]
+; CHECK-GI-NEXT: and w17, w18, #0x1
+; CHECK-GI-NEXT: umov w18, v0.b[5]
+; CHECK-GI-NEXT: umov w14, v0.b[6]
; CHECK-GI-NEXT: orr w10, w11, w10, lsl #4
; CHECK-GI-NEXT: orr w12, w12, w17, lsl #3
-; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: umov w11, v0.b[7]
; CHECK-GI-NEXT: and w16, w16, #0x1
-; CHECK-GI-NEXT: umov.b w17, v0[6]
+; CHECK-GI-NEXT: umov w17, v0.b[6]
; CHECK-GI-NEXT: orr w10, w10, w15, lsl #5
-; CHECK-GI-NEXT: umov.b w15, v0[7]
+; CHECK-GI-NEXT: umov w15, v0.b[7]
; CHECK-GI-NEXT: orr w12, w12, w16, lsl #4
-; CHECK-GI-NEXT: and w16, w0, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[7]
+; CHECK-GI-NEXT: and w16, w18, #0x1
+; CHECK-GI-NEXT: umov w18, v0.b[7]
; CHECK-GI-NEXT: and w14, w14, #0x1
; CHECK-GI-NEXT: orr w12, w12, w16, lsl #5
; CHECK-GI-NEXT: orr w10, w10, w14, lsl #6
; CHECK-GI-NEXT: and w11, w11, #0x1
; CHECK-GI-NEXT: and w13, w17, #0x1
; CHECK-GI-NEXT: orr w9, w9, w11, lsl #7
-; CHECK-GI-NEXT: mov.s v0[1], v1[0]
+; CHECK-GI-NEXT: mov v0.s[1], v1.s[0]
; CHECK-GI-NEXT: orr w11, w12, w13, lsl #6
; CHECK-GI-NEXT: and w12, w15, #0x1
-; CHECK-GI-NEXT: ; kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: orr w8, w10, w12, lsl #7
-; CHECK-GI-NEXT: and w10, w0, #0x1
+; CHECK-GI-NEXT: and w10, w18, #0x1
; CHECK-GI-NEXT: strb w9, [sp, #13]
; CHECK-GI-NEXT: orr w9, w11, w10, lsl #7
; CHECK-GI-NEXT: strb w8, [sp, #14]
@@ -1188,22 +1542,51 @@ define <2 x i8> @vector_to_vector_cast(<16 x i1> %arg) nounwind {
; bitmask lowering to each half.
define <32 x i1> @bitmask_v32i8_split(<32 x i8> %a, <32 x i8> %b) {
; CHECK-LABEL: bitmask_v32i8_split:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: cmeq.16b v1, v1, v3
-; CHECK-NEXT: adrp x9, lCPI21_0 at PAGE
-; CHECK-NEXT: cmeq.16b v0, v0, v2
-; CHECK-NEXT: ldr q2, [x9, lCPI21_0 at PAGEOFF]
-; CHECK-NEXT: and.16b v1, v1, v2
-; CHECK-NEXT: and.16b v0, v0, v2
-; CHECK-NEXT: addp.16b v1, v1, v1
-; CHECK-NEXT: addp.16b v0, v0, v0
-; CHECK-NEXT: addp.16b v1, v1, v1
-; CHECK-NEXT: addp.16b v0, v0, v0
-; CHECK-NEXT: addp.16b v1, v1, v1
-; CHECK-NEXT: addp.16b v0, v0, v0
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmeq v1.16b, v1.16b, v3.16b
+; CHECK-NEXT: adrp x9, .LCPI21_0
+; CHECK-NEXT: cmeq v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI21_0]
+; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT: str h1, [x8, #2]
; CHECK-NEXT: str h0, [x8]
; CHECK-NEXT: ret
+;
+; CHECK-BE-LABEL: bitmask_v32i8_split:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: adrp x9, .LCPI21_0
+; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI21_0
+; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v1.16b, v1.16b, v3.16b
+; CHECK-BE-NEXT: ld1 { v3.16b }, [x9]
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: and v1.16b, v1.16b, v3.16b
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: str h1, [x8, #2]
+; CHECK-BE-NEXT: str h0, [x8]
+; CHECK-BE-NEXT: ret
%r = icmp eq <32 x i8> %a, %b
ret <32 x i1> %r
}
@@ -1211,38 +1594,77 @@ define <32 x i1> @bitmask_v32i8_split(<32 x i8> %a, <32 x i8> %b) {
declare void @overflow()
define void @example.safeAdd(ptr %0, ptr %1, ptr %2) {
; CHECK-LABEL: example.safeAdd:
-; CHECK: ; %bb.0: ; %Entry
+; CHECK: // %bb.0: // %Entry
; CHECK-NEXT: ldp q2, q3, [x1]
-; CHECK-NEXT: adrp x8, lCPI22_0 at PAGE
+; CHECK-NEXT: adrp x8, .LCPI22_0
; CHECK-NEXT: ldp q1, q0, [x2]
-; CHECK-NEXT: ldr q4, [x8, lCPI22_0 at PAGEOFF]
-; CHECK-NEXT: add.16b v0, v3, v0
-; CHECK-NEXT: add.16b v1, v2, v1
-; CHECK-NEXT: cmhi.16b v3, v3, v0
-; CHECK-NEXT: cmhi.16b v2, v2, v1
-; CHECK-NEXT: and.16b v3, v3, v4
-; CHECK-NEXT: and.16b v2, v2, v4
-; CHECK-NEXT: addp.16b v3, v3, v3
-; CHECK-NEXT: addp.16b v2, v2, v2
-; CHECK-NEXT: addp.16b v3, v3, v3
-; CHECK-NEXT: addp.16b v2, v2, v2
-; CHECK-NEXT: addp.16b v3, v3, v3
-; CHECK-NEXT: addp.16b v2, v2, v2
-; CHECK-NEXT: umov.h w8, v3[0]
-; CHECK-NEXT: umov.h w9, v2[0]
+; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI22_0]
+; CHECK-NEXT: add v0.16b, v3.16b, v0.16b
+; CHECK-NEXT: add v1.16b, v2.16b, v1.16b
+; CHECK-NEXT: cmhi v3.16b, v3.16b, v0.16b
+; CHECK-NEXT: cmhi v2.16b, v2.16b, v1.16b
+; CHECK-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-NEXT: umov w8, v3.h[0]
+; CHECK-NEXT: umov w9, v2.h[0]
; CHECK-NEXT: orr w8, w9, w8
; CHECK-NEXT: tst w8, #0xffff
-; CHECK-NEXT: b.ne LBB22_2
-; CHECK-NEXT: ; %bb.1: ; %Else
+; CHECK-NEXT: b.ne .LBB22_2
+; CHECK-NEXT: // %bb.1: // %Else
; CHECK-NEXT: stp q1, q0, [x0]
; CHECK-NEXT: ret
-; CHECK-NEXT: LBB22_2: ; %Then
-; CHECK-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill
+; CHECK-NEXT: .LBB22_2: // %Then
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset w30, -8
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: bl _overflow
-; CHECK-NEXT: brk #0x1
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl overflow
+;
+; CHECK-BE-LABEL: example.safeAdd:
+; CHECK-BE: // %bb.0: // %Entry
+; CHECK-BE-NEXT: add x8, x1, #16
+; CHECK-BE-NEXT: ld1 { v3.16b }, [x1]
+; CHECK-BE-NEXT: ld1 { v0.16b }, [x2]
+; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-NEXT: add x8, x2, #16
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: adrp x8, .LCPI22_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
+; CHECK-BE-NEXT: add v0.16b, v3.16b, v0.16b
+; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-NEXT: add v1.16b, v2.16b, v1.16b
+; CHECK-BE-NEXT: cmhi v3.16b, v3.16b, v0.16b
+; CHECK-BE-NEXT: cmhi v2.16b, v2.16b, v1.16b
+; CHECK-BE-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-BE-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: rev16 v3.16b, v3.16b
+; CHECK-BE-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-NEXT: umov w8, v3.h[0]
+; CHECK-BE-NEXT: umov w9, v2.h[0]
+; CHECK-BE-NEXT: orr w8, w9, w8
+; CHECK-BE-NEXT: tst w8, #0xffff
+; CHECK-BE-NEXT: b.ne .LBB22_2
+; CHECK-BE-NEXT: // %bb.1: // %Else
+; CHECK-BE-NEXT: add x8, x0, #16
+; CHECK-BE-NEXT: st1 { v0.16b }, [x0]
+; CHECK-BE-NEXT: st1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ret
+; CHECK-BE-NEXT: .LBB22_2: // %Then
+; CHECK-BE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: .cfi_offset w30, -16
+; CHECK-BE-NEXT: bl overflow
Entry:
%3 = load <32 x i8>, ptr %1, align 16
%4 = load <32 x i8>, ptr %2, align 16
@@ -1271,17 +1693,17 @@ Else:
; 256-bit source, 4 lanes: narrowed <4 x i64> -> <4 x i32>.
define i4 @bitmask_v4i64(<4 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v4i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI23_0
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI23_0]
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv s0, v0.4s
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-SD-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI23_0
+; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI23_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v4i64:
; CHECK-BE: // %bb.0:
@@ -1297,6 +1719,28 @@ define i4 @bitmask_v4i64(<4 x i64> %v) {
; CHECK-BE-NEXT: addv s0, v0.4s
; CHECK-BE-NEXT: fmov w0, s0
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.2d, v0.2d, v0.2d
+; CHECK-GI-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: mov w9, v0.s[2]
+; CHECK-GI-NEXT: fmov w11, s0
+; CHECK-GI-NEXT: mov w10, v0.s[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w11, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w9, #0x1
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w11, w8, lsl #2
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <4 x i64> %v, zeroinitializer
%b = bitcast <4 x i1> %c to i4
ret i4 %b
@@ -1304,17 +1748,17 @@ define i4 @bitmask_v4i64(<4 x i64> %v) {
; 256-bit source, 8 lanes: narrowed <8 x i32> -> <8 x i16>.
define i8 @bitmask_v8i32(<8 x i32> %v) {
-; CHECK-LE-LABEL: bitmask_v8i32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI24_0
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI24_0]
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v8i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI24_0
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI24_0]
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v8i32:
; CHECK-BE: // %bb.0:
@@ -1332,6 +1776,41 @@ define i8 @bitmask_v8i32(<8 x i32> %v) {
; CHECK-BE-NEXT: addv h0, v0.8h
; CHECK-BE-NEXT: fmov w0, s0
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v8i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <8 x i32> %v, zeroinitializer
%b = bitcast <8 x i1> %c to i8
ret i8 %b
@@ -1339,21 +1818,21 @@ define i8 @bitmask_v8i32(<8 x i32> %v) {
; 512-bit source, 8 lanes: narrowed <8 x i64> -> <8 x i16>.
define i8 @bitmask_v8i64(<8 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v8i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI25_0
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI25_0]
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v8i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-SD-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI25_0
+; CHECK-SD-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-SD-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI25_0]
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v8i64:
; CHECK-BE: // %bb.0:
@@ -1375,6 +1854,45 @@ define i8 @bitmask_v8i64(<8 x i64> %v) {
; CHECK-BE-NEXT: addv h0, v0.8h
; CHECK-BE-NEXT: fmov w0, s0
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v8i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.2d, v0.2d, v0.2d
+; CHECK-GI-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; CHECK-GI-NEXT: cmtst v2.2d, v2.2d, v2.2d
+; CHECK-GI-NEXT: cmtst v3.2d, v3.2d, v3.2d
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <8 x i64> %v, zeroinitializer
%b = bitcast <8 x i1> %c to i8
ret i8 %b
@@ -1382,23 +1900,23 @@ define i8 @bitmask_v8i64(<8 x i64> %v) {
; 512-bit source, 16 lanes: narrowed <16 x i32> -> <16 x i8>.
define i16 @bitmask_v16i32(<16 x i32> %v) {
-; CHECK-LE-LABEL: bitmask_v16i32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v3.4s, v3.4s, #0
-; CHECK-LE-NEXT: cmeq v2.4s, v2.4s, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI26_0
-; CHECK-LE-NEXT: cmeq v1.4s, v1.4s, #0
-; CHECK-LE-NEXT: cmeq v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI26_0]
-; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w0, v0.h[0]
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v16i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v3.4s, v3.4s, #0
+; CHECK-SD-NEXT: cmeq v2.4s, v2.4s, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI26_0
+; CHECK-SD-NEXT: cmeq v1.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI26_0]
+; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: umov w0, v0.h[0]
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v16i32:
; CHECK-BE: // %bb.0:
@@ -1427,6 +1945,68 @@ define i16 @bitmask_v16i32(<16 x i32> %v) {
; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
; CHECK-BE-NEXT: umov w0, v0.h[0]
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v16i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: cmtst v2.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: cmtst v3.4s, v3.4s, v3.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[8]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[9]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[10]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[11]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[12]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[13]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[14]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[15]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
+; CHECK-GI-NEXT: strh w8, [sp, #14]
+; CHECK-GI-NEXT: and w0, w8, #0xffff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <16 x i32> %v, zeroinitializer
%b = bitcast <16 x i1> %c to i16
ret i16 %b
@@ -1435,35 +2015,35 @@ define i16 @bitmask_v16i32(<16 x i32> %v) {
; 1024-bit source, 16 lanes: narrowed <16 x i64> -> <16 x i8> (maximum
; narrowing ratio: 8x).
define i16 @bitmask_v16i64(<16 x i64> %v) {
-; CHECK-LE-LABEL: bitmask_v16i64:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: cmeq v7.2d, v7.2d, #0
-; CHECK-LE-NEXT: cmeq v6.2d, v6.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI27_0
-; CHECK-LE-NEXT: cmeq v5.2d, v5.2d, #0
-; CHECK-LE-NEXT: cmeq v4.2d, v4.2d, #0
-; CHECK-LE-NEXT: cmeq v3.2d, v3.2d, #0
-; CHECK-LE-NEXT: cmeq v2.2d, v2.2d, #0
-; CHECK-LE-NEXT: cmeq v1.2d, v1.2d, #0
-; CHECK-LE-NEXT: cmeq v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-LE-NEXT: uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-LE-NEXT: uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: uzp1 v1.8h, v4.8h, v6.8h
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-LE-NEXT: mvn v1.16b, v1.16b
-; CHECK-LE-NEXT: mvn v0.16b, v0.16b
-; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI27_0]
-; CHECK-LE-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w0, v0.h[0]
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v16i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: cmeq v7.2d, v7.2d, #0
+; CHECK-SD-NEXT: cmeq v6.2d, v6.2d, #0
+; CHECK-SD-NEXT: adrp x8, .LCPI27_0
+; CHECK-SD-NEXT: cmeq v5.2d, v5.2d, #0
+; CHECK-SD-NEXT: cmeq v4.2d, v4.2d, #0
+; CHECK-SD-NEXT: cmeq v3.2d, v3.2d, #0
+; CHECK-SD-NEXT: cmeq v2.2d, v2.2d, #0
+; CHECK-SD-NEXT: cmeq v1.2d, v1.2d, #0
+; CHECK-SD-NEXT: cmeq v0.2d, v0.2d, #0
+; CHECK-SD-NEXT: uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-SD-NEXT: uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-SD-NEXT: uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT: mvn v1.16b, v1.16b
+; CHECK-SD-NEXT: mvn v0.16b, v0.16b
+; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI27_0]
+; CHECK-SD-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-SD-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT: umov w0, v0.h[0]
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v16i64:
; CHECK-BE: // %bb.0:
@@ -1504,6 +2084,76 @@ define i16 @bitmask_v16i64(<16 x i64> %v) {
; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
; CHECK-BE-NEXT: umov w0, v0.h[0]
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v16i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.2d, v0.2d, v0.2d
+; CHECK-GI-NEXT: cmtst v1.2d, v1.2d, v1.2d
+; CHECK-GI-NEXT: cmtst v2.2d, v2.2d, v2.2d
+; CHECK-GI-NEXT: cmtst v3.2d, v3.2d, v3.2d
+; CHECK-GI-NEXT: cmtst v4.2d, v4.2d, v4.2d
+; CHECK-GI-NEXT: cmtst v5.2d, v5.2d, v5.2d
+; CHECK-GI-NEXT: cmtst v6.2d, v6.2d, v6.2d
+; CHECK-GI-NEXT: cmtst v7.2d, v7.2d, v7.2d
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-GI-NEXT: uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[8]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[9]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[10]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[11]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[12]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[13]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[14]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[15]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
+; CHECK-GI-NEXT: strh w8, [sp, #14]
+; CHECK-GI-NEXT: and w0, w8, #0xffff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <16 x i64> %v, zeroinitializer
%b = bitcast <16 x i1> %c to i16
ret i16 %b
@@ -1511,17 +2161,17 @@ define i16 @bitmask_v16i64(<16 x i64> %v) {
; Float compares from a wide source take the same path.
define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
-; CHECK-LE-LABEL: bitmask_v8f32:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: fcmeq v1.4s, v1.4s, v3.4s
-; CHECK-LE-NEXT: fcmeq v0.4s, v0.4s, v2.4s
-; CHECK-LE-NEXT: adrp x8, .LCPI28_0
-; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI28_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v8f32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: adrp x8, .LCPI28_0
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI28_0]
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v8f32:
; CHECK-BE: // %bb.0:
@@ -1543,6 +2193,41 @@ define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
; CHECK-BE-NEXT: addv h0, v0.8h
; CHECK-BE-NEXT: fmov w0, s0
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v8f32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: fcmeq v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: fcmeq v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = fcmp oeq <8 x float> %a, %b
%m = bitcast <8 x i1> %c to i8
ret i8 %m
@@ -1552,22 +2237,22 @@ define i8 @bitmask_v8f32(<8 x float> %a, <8 x float> %b) {
; register-only (predicate -> Z register -> narrow -> weights+reduce).
; TODO: a predicated UADDV over the weight vector would shorten this further.
define i8 @bitmask_v8i32_sve(<8 x i32> %v) vscale_range(2,2) "target-features"="+sve" {
-; CHECK-LE-LABEL: bitmask_v8i32_sve:
-; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: ptrue p0.s, vl4
-; CHECK-LE-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-LE-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-LE-NEXT: adrp x8, .LCPI29_0
-; CHECK-LE-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-LE-NEXT: ptrue p0.s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI29_0]
-; CHECK-LE-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; CHECK-LE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-LE-NEXT: uzp1 z0.h, z0.h, z0.h
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w0, s0
-; CHECK-LE-NEXT: ret
+; CHECK-SD-LABEL: bitmask_v8i32_sve:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ptrue p0.s, vl4
+; CHECK-SD-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SD-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-SD-NEXT: adrp x8, .LCPI29_0
+; CHECK-SD-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-SD-NEXT: ptrue p0.s
+; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI29_0]
+; CHECK-SD-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-SD-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SD-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: addv h0, v0.8h
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
;
; CHECK-BE-LABEL: bitmask_v8i32_sve:
; CHECK-BE: // %bb.0:
@@ -1591,6 +2276,41 @@ define i8 @bitmask_v8i32_sve(<8 x i32> %v) vscale_range(2,2) "target-features"="
; CHECK-BE-NEXT: addv h0, v0.8h
; CHECK-BE-NEXT: fmov w0, s0
; CHECK-BE-NEXT: ret
+;
+; CHECK-GI-LABEL: bitmask_v8i32_sve:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT: cmtst v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: xtn v0.8b, v0.8h
+; CHECK-GI-NEXT: umov w8, v0.b[1]
+; CHECK-GI-NEXT: umov w9, v0.b[0]
+; CHECK-GI-NEXT: umov w10, v0.b[2]
+; CHECK-GI-NEXT: umov w11, v0.b[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov w10, v0.b[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov w11, v0.b[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%c = icmp ne <8 x i32> %v, zeroinitializer
%b = bitcast <8 x i1> %c to i8
ret i8 %b
More information about the llvm-commits
mailing list