[llvm] [AArch64] Lower cttz(bitcast <Nxi1> to iN) with shrn-based compressed movemask (PR #199081)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 15:58:33 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/199081
>From 347c332815aec730ba0b57cfbc13f9f67dd5141d Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 29 Jun 2026 00:43:36 +0000
Subject: [PATCH 1/4] [AArch64][NFC] Add tests for cttz(bitcast <N x i1>)
lowering
---
.../AArch64/cttz-of-bool-vector-bitcast.ll | 986 ++++++++++++++++++
1 file changed, 986 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
diff --git a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
new file mode 100644
index 0000000000000..5c213cc02e3db
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
@@ -0,0 +1,986 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i4 @llvm.cttz.i4(i4, i1)
+declare i2 @llvm.cttz.i2(i2, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
+define i16 @cttz_v16i8(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI0_0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: orr w8, w8, #0x10000
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI0_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: orr w8, w8, #0x10000
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i16(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI1_0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x100
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: adrp x8, .LCPI1_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x100
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i8 @cttz_v8i8(<8 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v8i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI2_0
+; CHECK-LE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: addv b0, v0.8b
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x100
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-NEXT: adrp x8, .LCPI2_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-BE-NEXT: ld1 { v1.8b }, [x8]
+; CHECK-BE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv b0, v0.8b
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x100
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i8> %v, splat (i8 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i16 @cttz_v16i8_with_any_check(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: adrp x8, .LCPI3_0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: orr w8, w8, #0x10000
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI3_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: orr w8, w8, #0x10000
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %any = icmp ne i16 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+notfound:
+ ret i16 16
+}
+
+
+define i4 @cttz_v4i32(<4 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI4_0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv s0, v0.4s
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x10
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI4_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI4_0
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x10
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i32> %v, splat (i32 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i2 @cttz_v2i64(<2 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v2i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI5_0
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp d0, v0.2d
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x4
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI5_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI5_0
+; CHECK-BE-NEXT: ld1 { v1.2d }, [x8]
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp d0, v0.2d
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x4
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i64> %v, splat (i64 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+
+define i16 @cttz_v16i8_eq(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_eq:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #42
+; CHECK-LE-NEXT: adrp x8, .LCPI6_0
+; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI6_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: orr w8, w8, #0x10000
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_eq:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #42
+; CHECK-BE-NEXT: adrp x8, .LCPI6_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: orr w8, w8, #0x10000
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp eq <16 x i8> %v, splat (i8 42)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_ne(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_ne:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI7_0
+; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
+; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: orr w8, w8, #0x10000
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_ne:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI7_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI7_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: orr w8, w8, #0x10000
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp ne <16 x i8> %v, zeroinitializer
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_sgt(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_sgt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #5
+; CHECK-LE-NEXT: adrp x8, .LCPI8_0
+; CHECK-LE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI8_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: orr w8, w8, #0x10000
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_sgt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #5
+; CHECK-BE-NEXT: adrp x8, .LCPI8_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI8_0
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: orr w8, w8, #0x10000
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp sgt <16 x i8> %v, splat (i8 5)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+
+define i4 @cttz_v4f32_olt(<4 x float> %v) {
+; CHECK-LE-LABEL: cttz_v4f32_olt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-LE-NEXT: adrp x8, .LCPI9_0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI9_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv s0, v0.4s
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x10
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4f32_olt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: adrp x8, .LCPI9_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI9_0
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x10
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = fcmp olt <4 x float> %v, zeroinitializer
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+
+define i8 @cttz_v8i16_with_any_check(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: adrp x8, .LCPI10_0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x100
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: adrp x8, .LCPI10_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI10_0
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x100
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %any = icmp ne i8 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 true)
+ ret i8 %ctz
+notfound:
+ ret i8 8
+}
+
+
+define i16 @cttz_v16i8_zero_undef(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI11_0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI11_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_0
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+}
+
+
+define i32 @cttz_plain_i32(i32 %x) {
+; CHECK-LABEL: cttz_plain_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rbit w8, w0
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %ctz = call i32 @llvm.cttz.i32(i32 %x, i1 false)
+ ret i32 %ctz
+}
+
+define i32 @cttz_bitcast_from_float(float %f) {
+; CHECK-LABEL: cttz_bitcast_from_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: rbit w8, w8
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %i = bitcast float %f to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %i, i1 false)
+ ret i32 %ctz
+}
+
+define i3 @cttz_v3i8_non_pow2_lanes(<3 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: fmov s0, w0
+; CHECK-LE-NEXT: mov v0.h[1], w1
+; CHECK-LE-NEXT: mov v0.h[2], w2
+; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: umov w9, v0.h[1]
+; CHECK-LE-NEXT: umov w10, v0.h[2]
+; CHECK-LE-NEXT: and w8, w8, #0x1
+; CHECK-LE-NEXT: bfi w8, w9, #1, #1
+; CHECK-LE-NEXT: bfi w8, w10, #2, #1
+; CHECK-LE-NEXT: orr w8, w8, #0x8
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: fmov s0, w0
+; CHECK-BE-NEXT: mov v0.h[1], w1
+; CHECK-BE-NEXT: mov v0.h[2], w2
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: umov w8, v0.h[1]
+; CHECK-BE-NEXT: umov w9, v0.h[0]
+; CHECK-BE-NEXT: umov w10, v0.h[2]
+; CHECK-BE-NEXT: ubfiz w8, w8, #1, #1
+; CHECK-BE-NEXT: bfi w8, w9, #2, #1
+; CHECK-BE-NEXT: bfxil w8, w10, #0, #1
+; CHECK-BE-NEXT: orr w8, w8, #0x8
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <3 x i8> %v, splat (i8 0)
+ %bm = bitcast <3 x i1> %cmp to i3
+ %ctz = call i3 @llvm.cttz.i3(i3 %bm, i1 false)
+ ret i3 %ctz
+}
+declare i3 @llvm.cttz.i3(i3, i1)
+
+define i16 @cttz_v16i16(<16 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v16i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI15_0
+; CHECK-LE-NEXT: mov w9, #65536 // =0x10000
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI15_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: bfxil w9, w8, #0, #16
+; CHECK-LE-NEXT: rbit w8, w9
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: adrp x8, .LCPI15_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI15_0
+; CHECK-BE-NEXT: mov w9, #65536 // =0x10000
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: bfxil w9, w8, #0, #16
+; CHECK-BE-NEXT: rbit w8, w9
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i16> %v, splat (i16 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i32(<8 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v8i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI16_0
+; CHECK-LE-NEXT: mov w9, #256 // =0x100
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv h0, v0.8h
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: bfxil w9, w8, #0, #8
+; CHECK-LE-NEXT: rbit w8, w9
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: adrp x9, .LCPI16_0
+; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI16_0
+; CHECK-BE-NEXT: mov w8, #256 // =0x100
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: ld1 { v1.8h }, [x9]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv h0, v0.8h
+; CHECK-BE-NEXT: fmov w9, s0
+; CHECK-BE-NEXT: bfxil w8, w9, #0, #8
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i32> %v, splat (i32 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i4 @cttz_v4i64(<4 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v4i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: adrp x8, .LCPI17_0
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI17_0]
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: addv s0, v0.4s
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x10
+; CHECK-LE-NEXT: and w8, w8, #0xff
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: adrp x8, .LCPI17_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI17_0
+; CHECK-BE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: addv s0, v0.4s
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x10
+; CHECK-BE-NEXT: and w8, w8, #0xff
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i64> %v, splat (i64 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i32 @cttz_v32i8(<32 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v32i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI18_0
+; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q2, [x8, :lo12:.LCPI18_0]
+; CHECK-LE-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v1.h[0]
+; CHECK-LE-NEXT: umov w9, v0.h[0]
+; CHECK-LE-NEXT: bfi w9, w8, #16, #16
+; CHECK-LE-NEXT: rbit w8, w9
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v32i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI18_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI18_0
+; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: and v1.16b, v1.16b, v2.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v1.16b, v1.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: umov w9, v1.h[0]
+; CHECK-BE-NEXT: bfi w9, w8, #16, #16
+; CHECK-BE-NEXT: rbit w8, w9
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <32 x i8> %v, splat (i8 0)
+ %bm = bitcast <32 x i1> %cmp to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %bm, i1 false)
+ ret i32 %ctz
+}
+
+define i64 @cttz_v64i8_too_many_lanes(<64 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v64i8_too_many_lanes:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI19_0
+; CHECK-LE-NEXT: cmlt v3.16b, v3.16b, #0
+; CHECK-LE-NEXT: cmlt v2.16b, v2.16b, #0
+; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: ldr q4, [x8, :lo12:.LCPI19_0]
+; CHECK-LE-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-LE-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-LE-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-LE-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-LE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: umov w8, v3.h[0]
+; CHECK-LE-NEXT: umov w9, v2.h[0]
+; CHECK-LE-NEXT: umov w10, v1.h[0]
+; CHECK-LE-NEXT: umov w11, v0.h[0]
+; CHECK-LE-NEXT: bfi w9, w8, #16, #16
+; CHECK-LE-NEXT: bfi w11, w10, #16, #16
+; CHECK-LE-NEXT: orr x8, x11, x9, lsl #32
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x0, x8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v64i8_too_many_lanes:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev64 v3.16b, v3.16b
+; CHECK-BE-NEXT: adrp x8, .LCPI19_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI19_0
+; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: cmlt v3.16b, v3.16b, #0
+; CHECK-BE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-BE-NEXT: cmlt v2.16b, v2.16b, #0
+; CHECK-BE-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-BE-NEXT: and v3.16b, v3.16b, v4.16b
+; CHECK-BE-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-BE-NEXT: and v2.16b, v2.16b, v4.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-BE-NEXT: addp v2.16b, v2.16b, v2.16b
+; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev16 v3.16b, v3.16b
+; CHECK-BE-NEXT: rev16 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev16 v2.16b, v2.16b
+; CHECK-BE-NEXT: umov w8, v0.h[0]
+; CHECK-BE-NEXT: umov w11, v3.h[0]
+; CHECK-BE-NEXT: umov w9, v1.h[0]
+; CHECK-BE-NEXT: umov w10, v2.h[0]
+; CHECK-BE-NEXT: bfi w9, w8, #16, #16
+; CHECK-BE-NEXT: bfi w11, w10, #16, #16
+; CHECK-BE-NEXT: orr x8, x11, x9, lsl #32
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x0, x8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <64 x i8> %v, splat (i8 0)
+ %bm = bitcast <64 x i1> %cmp to i64
+ %ctz = call i64 @llvm.cttz.i64(i64 %bm, i1 false)
+ ret i64 %ctz
+}
+
+define i4 @cttz_v4i16(<4 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v4i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: adrp x8, .LCPI20_0
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI20_0]
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: addv h0, v0.4h
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x10
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: adrp x8, .LCPI20_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_0
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x10
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i16> %v, splat (i16 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i2 @cttz_v2i32(<2 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v2i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: mov x8, #1 // =0x1
+; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-LE-NEXT: movk x8, #2, lsl #32
+; CHECK-LE-NEXT: fmov d1, x8
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x4
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: adrp x8, .LCPI21_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_0
+; CHECK-BE-NEXT: ld1 { v1.2s }, [x8]
+; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x4
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i32> %v, splat (i32 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+define i2 @cttz_v2i8(<2 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v2i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-LE-NEXT: mov x8, #1 // =0x1
+; CHECK-LE-NEXT: movk x8, #2, lsl #32
+; CHECK-LE-NEXT: fmov d1, x8
+; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x4
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
+; CHECK-BE-NEXT: adrp x8, .LCPI22_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
+; CHECK-BE-NEXT: ld1 { v1.2s }, [x8]
+; CHECK-BE-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addp v0.2s, v0.2s, v0.2s
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x4
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i8> %v, splat (i8 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+define i4 @cttz_v4i8(<4 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v4i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: adrp x8, .LCPI23_0
+; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI23_0]
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-LE-NEXT: addv h0, v0.4h
+; CHECK-LE-NEXT: fmov w8, s0
+; CHECK-LE-NEXT: orr w8, w8, #0x10
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
+; CHECK-BE-NEXT: adrp x8, .LCPI23_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI23_0
+; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-BE-NEXT: addv h0, v0.4h
+; CHECK-BE-NEXT: fmov w8, s0
+; CHECK-BE-NEXT: orr w8, w8, #0x10
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i8> %v, splat (i8 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
>From 4d7eb5c329f3f2630f7353c26433394c203a7f45 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 29 Jun 2026 00:43:47 +0000
Subject: [PATCH 2/4] [AArch64] Lower cttz(bitcast <N x i1> to iN) with
shrn-based compressed movemask
---
.../Target/AArch64/AArch64ISelLowering.cpp | 131 ++++
.../AArch64/cttz-of-bool-vector-bitcast.ll | 672 +++++++-----------
2 files changed, 383 insertions(+), 420 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..d16bb609fec81 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1234,6 +1234,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::GlobalAddress);
setTargetDAGCombine(ISD::CTLZ);
+ setTargetDAGCombine({ISD::CTTZ, ISD::CTTZ_ZERO_POISON});
setTargetDAGCombine(ISD::GET_ACTIVE_LANE_MASK);
@@ -26715,6 +26716,133 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits);
}
+// When taking the trailing-zero count of a bitcast from <N x i1> to scalar
+// iN, we can pack each lane into a small bit-slot of a 64-bit scalar
+// with shrn (for byte lanes) or xtn (for wider lanes) then run a
+// scalar cttz on the result.
+static SDValue performCTTZCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG) {
+ // Run before vectorToScalarBitmask() expands it.
+ if (!DCI.isBeforeLegalize())
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ if (!VT.isScalarInteger())
+ return SDValue();
+ SDValue BitcastNode = N->getOperand(0);
+ if (BitcastNode.getOpcode() != ISD::BITCAST)
+ return SDValue();
+ SDValue Predicate = BitcastNode.getOperand(0);
+ EVT PredVT = Predicate.getValueType();
+ if (!PredVT.isFixedLengthVector() || PredVT.getVectorElementType() != MVT::i1)
+ return SDValue();
+
+ unsigned NumLanes = PredVT.getVectorNumElements();
+ if (!isPowerOf2_32(NumLanes) || NumLanes < 2 || NumLanes > 32)
+ return SDValue();
+
+ constexpr unsigned DRegBits = 64;
+ constexpr unsigned QRegBits = 128;
+ constexpr unsigned MinLaneBits = 8;
+
+ // Pick a lane width to sext the predicate to. Prefer the original
+ // compare input width if we can find it and it fits in at most two NEON
+ // registers but otherwise pick a minimum-width type that fits.
+ EVT MaskVT = tryGetOriginalBoolVectorType(Predicate);
+ if (!MaskVT.isSimple() ||
+ MaskVT.changeVectorElementTypeToInteger().getSizeInBits() >
+ 2 * QRegBits ||
+ MaskVT.changeVectorElementTypeToInteger().getSizeInBits() < DRegBits) {
+ unsigned LaneBits = std::max(DRegBits / NumLanes, MinLaneBits);
+ MaskVT = MVT::getVectorVT(MVT::getIntegerVT(LaneBits), NumLanes);
+ }
+ MaskVT = MaskVT.changeVectorElementTypeToInteger();
+
+ unsigned MaskBits = MaskVT.getSizeInBits();
+ if (MaskBits != DRegBits && MaskBits != QRegBits && MaskBits != 2 * QRegBits)
+ return SDValue();
+
+ // Target 64 bits when lanes fit, otherwise 128.
+ unsigned TargetBits = DRegBits;
+ if (MaskBits == 2 * QRegBits && NumLanes > 16)
+ TargetBits = QRegBits;
+ unsigned CompressedBits = std::min(MaskBits, TargetBits);
+
+ SDLoc DL(N);
+ SDValue Mask = DAG.getSExtOrTrunc(Predicate, DL, MaskVT);
+
+ LLVMContext &Ctx = *DAG.getContext();
+
+ // NEON's smallest vector lane type is i8 so we can't narrow further.
+ // Instead, bitcast the mask to i16 lanes and shrn by 4 to pack each
+ // input byte into 4 bits of the output
+ auto ShrnPair = [&](SDValue V) -> SDValue {
+ EVT InVT = V.getValueType();
+ EVT OutVT = InVT.getHalfNumVectorElementsVT(Ctx);
+ EVT PairedVT = OutVT.widenIntegerVectorElementType(Ctx);
+ SDValue Paired = DAG.getNode(AArch64ISD::NVCAST, DL, PairedVT, V);
+ SDValue ShAmt = DAG.getConstant(MinLaneBits / 2, DL, PairedVT);
+ SDValue Shifted = DAG.getNode(ISD::SRL, DL, PairedVT, Paired, ShAmt);
+ return DAG.getNode(ISD::TRUNCATE, DL, OutVT, Shifted);
+ };
+
+ // Trailing-zero count of the compressed result as an i64.
+ auto CompressedCttz = [&](SDValue V) -> SDValue {
+ if (V.getValueType().getSizeInBits() == DRegBits) {
+ SDValue Nv = DAG.getNode(AArch64ISD::NVCAST, DL, MVT::v1i64, V);
+ SDValue Scalar = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Nv,
+ DAG.getConstant(0, DL, MVT::i64));
+ return DAG.getNode(ISD::CTTZ, DL, MVT::i64, Scalar);
+ }
+ SDValue Compressed = DAG.getBitcast(MVT::i128, V);
+ // BITCAST puts lane 0 at the high end of the scalar on BE (unlike NVCAST
+ // above), so use clz instead of cttz to find the first match
+ unsigned Op = DAG.getDataLayout().isLittleEndian() ? ISD::CTTZ : ISD::CTLZ;
+ SDValue Ctz = DAG.getNode(Op, DL, MVT::i128, Compressed);
+ return DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Ctz);
+ };
+
+ // Narrow toward CompressedBits. shrn-pair is only used once since it
+ // leaves the i8 lanes already packed.
+ SDValue Cur = Mask;
+ bool UsedShrnPair = false;
+ while (Cur.getValueType().getSizeInBits() > CompressedBits) {
+ EVT CurVT = Cur.getValueType();
+ unsigned CurLaneBits = CurVT.getScalarSizeInBits();
+ if (CurLaneBits == MinLaneBits) {
+ if (UsedShrnPair)
+ break;
+ if (CurVT.getSizeInBits() <= QRegBits) {
+ Cur = ShrnPair(Cur);
+ } else {
+ // Split into Q-reg-sized halves so each ShrnPair stays in range.
+ EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx);
+ SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(0, DL, MVT::i64));
+ SDValue Hi = DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64));
+ Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo),
+ ShrnPair(Hi));
+ }
+ UsedShrnPair = true;
+ } else {
+ // For wider lanes, xtn each lane down to half its width.
+ EVT NarrowVT = CurVT.changeVectorElementType(
+ Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2));
+ Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur);
+ }
+ }
+
+ SDValue Ctz = CompressedCttz(Cur);
+ unsigned LaneIndexShift = Log2_32(CompressedBits / NumLanes);
+ if (LaneIndexShift)
+ Ctz = DAG.getNode(ISD::SRL, DL, MVT::i64, Ctz,
+ DAG.getShiftAmountConstant(LaneIndexShift, MVT::i64, DL));
+ return DAG.getSExtOrTrunc(Ctz, DL, VT);
+}
+
static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG,
StoreSDNode *Store) {
if (!Store->isTruncatingStore())
@@ -30280,6 +30408,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performMINMAXCombine(N, DAG, *this);
case ISD::TRUNCATE:
return performTruncateCombine(N, DAG, DCI);
+ case ISD::CTTZ:
+ case ISD::CTTZ_ZERO_POISON:
+ return performCTTZCombine(N, DCI, DAG);
case AArch64ISD::ANDS:
return performANDSCombine(N, DCI);
case AArch64ISD::ADC:
diff --git a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
index 5c213cc02e3db..dfded15396df7 100644
--- a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
@@ -12,36 +12,26 @@ declare i64 @llvm.cttz.i64(i64, i1)
define i16 @cttz_v16i8(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI0_0
; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: orr w8, w8, #0x10000
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI0_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: orr w8, w8, #0x10000
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <16 x i8> %v, splat (i8 0)
%bm = bitcast <16 x i1> %cmp to i16
@@ -52,31 +42,26 @@ define i16 @cttz_v16i8(<16 x i8> %v) {
define i8 @cttz_v8i16(<8 x i16> %v) {
; CHECK-LE-LABEL: cttz_v8i16:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI1_0
; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x100
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v8i16:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-NEXT: adrp x8, .LCPI1_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI1_0
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x100
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <8 x i16> %v, splat (i16 0)
%bm = bitcast <8 x i1> %cmp to i8
@@ -87,30 +72,23 @@ define i8 @cttz_v8i16(<8 x i16> %v) {
define i8 @cttz_v8i8(<8 x i8> %v) {
; CHECK-LE-LABEL: cttz_v8i8:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI2_0
; CHECK-LE-NEXT: cmlt v0.8b, v0.8b, #0
-; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
-; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-LE-NEXT: addv b0, v0.8b
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x100
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v8i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
-; CHECK-BE-NEXT: adrp x8, .LCPI2_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI2_0
-; CHECK-BE-NEXT: ld1 { v1.8b }, [x8]
; CHECK-BE-NEXT: cmlt v0.8b, v0.8b, #0
-; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-BE-NEXT: addv b0, v0.8b
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x100
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <8 x i8> %v, splat (i8 0)
%bm = bitcast <8 x i1> %cmp to i8
@@ -121,36 +99,26 @@ define i8 @cttz_v8i8(<8 x i8> %v) {
define i16 @cttz_v16i8_with_any_check(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8_with_any_check:
; CHECK-LE: // %bb.0: // %common.ret
-; CHECK-LE-NEXT: adrp x8, .LCPI3_0
; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: orr w8, w8, #0x10000
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8_with_any_check:
; CHECK-BE: // %bb.0: // %common.ret
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI3_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI3_0
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: orr w8, w8, #0x10000
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <16 x i8> %v, splat (i8 0)
%bm = bitcast <16 x i1> %cmp to i16
@@ -167,31 +135,26 @@ notfound:
define i4 @cttz_v4i32(<4 x i32> %v) {
; CHECK-LE-LABEL: cttz_v4i32:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI4_0
; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI4_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv s0, v0.4s
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x10
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v4i32:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI4_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI4_0
-; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv s0, v0.4s
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x10
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <4 x i32> %v, splat (i32 0)
%bm = bitcast <4 x i1> %cmp to i4
@@ -202,30 +165,25 @@ define i4 @cttz_v4i32(<4 x i32> %v) {
define i2 @cttz_v2i64(<2 x i64> %v) {
; CHECK-LE-LABEL: cttz_v2i64:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI5_0
; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp d0, v0.2d
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x4
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: xtn v0.2s, v0.2d
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v2i64:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: adrp x8, .LCPI5_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI5_0
-; CHECK-BE-NEXT: ld1 { v1.2d }, [x8]
; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp d0, v0.2d
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x4
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: xtn v0.2s, v0.2d
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <2 x i64> %v, splat (i64 0)
%bm = bitcast <2 x i1> %cmp to i2
@@ -238,37 +196,27 @@ define i16 @cttz_v16i8_eq(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8_eq:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: movi v1.16b, #42
-; CHECK-LE-NEXT: adrp x8, .LCPI6_0
; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI6_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: orr w8, w8, #0x10000
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8_eq:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
; CHECK-BE-NEXT: movi v1.16b, #42
-; CHECK-BE-NEXT: adrp x8, .LCPI6_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: orr w8, w8, #0x10000
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp eq <16 x i8> %v, splat (i8 42)
%bm = bitcast <16 x i1> %cmp to i16
@@ -279,36 +227,26 @@ define i16 @cttz_v16i8_eq(<16 x i8> %v) {
define i16 @cttz_v16i8_ne(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8_ne:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI7_0
-; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI7_0]
-; CHECK-LE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: orr w8, w8, #0x10000
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8_ne:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI7_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI7_0
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, #0
-; CHECK-BE-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: orr w8, w8, #0x10000
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp ne <16 x i8> %v, zeroinitializer
%bm = bitcast <16 x i1> %cmp to i16
@@ -320,37 +258,27 @@ define i16 @cttz_v16i8_sgt(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8_sgt:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: movi v1.16b, #5
-; CHECK-LE-NEXT: adrp x8, .LCPI8_0
; CHECK-LE-NEXT: cmgt v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI8_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: orr w8, w8, #0x10000
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8_sgt:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
; CHECK-BE-NEXT: movi v1.16b, #5
-; CHECK-BE-NEXT: adrp x8, .LCPI8_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI8_0
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmgt v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: orr w8, w8, #0x10000
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp sgt <16 x i8> %v, splat (i8 5)
%bm = bitcast <16 x i1> %cmp to i16
@@ -363,30 +291,25 @@ define i4 @cttz_v4f32_olt(<4 x float> %v) {
; CHECK-LE-LABEL: cttz_v4f32_olt:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: fcmlt v0.4s, v0.4s, #0.0
-; CHECK-LE-NEXT: adrp x8, .LCPI9_0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI9_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv s0, v0.4s
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x10
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v4f32_olt:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
-; CHECK-BE-NEXT: adrp x8, .LCPI9_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI9_0
-; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: fcmlt v0.4s, v0.4s, #0.0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv s0, v0.4s
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x10
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = fcmp olt <4 x float> %v, zeroinitializer
%bm = bitcast <4 x i1> %cmp to i4
@@ -398,31 +321,26 @@ define i4 @cttz_v4f32_olt(<4 x float> %v) {
define i8 @cttz_v8i16_with_any_check(<8 x i16> %v) {
; CHECK-LE-LABEL: cttz_v8i16_with_any_check:
; CHECK-LE: // %bb.0: // %common.ret
-; CHECK-LE-NEXT: adrp x8, .LCPI10_0
; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI10_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x100
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v8i16_with_any_check:
; CHECK-BE: // %bb.0: // %common.ret
; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
-; CHECK-BE-NEXT: adrp x8, .LCPI10_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI10_0
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x100
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <8 x i16> %v, splat (i16 0)
%bm = bitcast <8 x i1> %cmp to i8
@@ -439,34 +357,26 @@ notfound:
define i16 @cttz_v16i8_zero_undef(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8_zero_undef:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI11_0
; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI11_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i8_zero_undef:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI11_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_0
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <16 x i8> %v, splat (i8 0)
%bm = bitcast <16 x i1> %cmp to i16
@@ -551,50 +461,32 @@ declare i3 @llvm.cttz.i3(i3, i1)
define i16 @cttz_v16i16(<16 x i16> %v) {
; CHECK-LE-LABEL: cttz_v16i16:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: sub sp, sp, #16
-; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI15_0
-; CHECK-LE-NEXT: mov w9, #65536 // =0x10000
; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI15_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v0.h[0]
-; CHECK-LE-NEXT: bfxil w9, w8, #0, #16
-; CHECK-LE-NEXT: rbit w8, w9
-; CHECK-LE-NEXT: clz w0, w8
-; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v16i16:
; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: sub sp, sp, #16
-; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
-; CHECK-BE-NEXT: adrp x8, .LCPI15_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI15_0
-; CHECK-BE-NEXT: mov w9, #65536 // =0x10000
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: bfxil w9, w8, #0, #16
-; CHECK-BE-NEXT: rbit w8, w9
-; CHECK-BE-NEXT: clz w0, w8
-; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <16 x i16> %v, splat (i16 0)
%bm = bitcast <16 x i1> %cmp to i16
@@ -605,45 +497,32 @@ define i16 @cttz_v16i16(<16 x i16> %v) {
define i8 @cttz_v8i32(<8 x i32> %v) {
; CHECK-LE-LABEL: cttz_v8i32:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: sub sp, sp, #16
-; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI16_0
-; CHECK-LE-NEXT: mov w9, #256 // =0x100
; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv h0, v0.8h
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: bfxil w9, w8, #0, #8
-; CHECK-LE-NEXT: rbit w8, w9
-; CHECK-LE-NEXT: clz w0, w8
-; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v8i32:
; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: sub sp, sp, #16
-; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
-; CHECK-BE-NEXT: adrp x9, .LCPI16_0
-; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI16_0
-; CHECK-BE-NEXT: mov w8, #256 // =0x100
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-BE-NEXT: ld1 { v1.8h }, [x9]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv h0, v0.8h
-; CHECK-BE-NEXT: fmov w9, s0
-; CHECK-BE-NEXT: bfxil w8, w9, #0, #8
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
-; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <8 x i32> %v, splat (i32 0)
%bm = bitcast <8 x i1> %cmp to i8
@@ -654,43 +533,30 @@ define i8 @cttz_v8i32(<8 x i32> %v) {
define i4 @cttz_v4i64(<4 x i64> %v) {
; CHECK-LE-LABEL: cttz_v4i64:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: sub sp, sp, #16
-; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
; CHECK-LE-NEXT: cmlt v1.2d, v1.2d, #0
; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
-; CHECK-LE-NEXT: adrp x8, .LCPI17_0
; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-LE-NEXT: ldr q1, [x8, :lo12:.LCPI17_0]
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-LE-NEXT: addv s0, v0.4s
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x10
-; CHECK-LE-NEXT: and w8, w8, #0xff
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
-; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v4i64:
; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: sub sp, sp, #16
-; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: adrp x8, .LCPI17_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI17_0
; CHECK-BE-NEXT: cmlt v1.2d, v1.2d, #0
; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-BE-NEXT: ld1 { v1.4s }, [x8]
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT: addv s0, v0.4s
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x10
-; CHECK-BE-NEXT: and w8, w8, #0xff
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
-; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <4 x i64> %v, splat (i64 0)
%bm = bitcast <4 x i1> %cmp to i4
@@ -701,51 +567,43 @@ define i4 @cttz_v4i64(<4 x i64> %v) {
define i32 @cttz_v32i8(<32 x i8> %v) {
; CHECK-LE-LABEL: cttz_v32i8:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI18_0
-; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q2, [x8, :lo12:.LCPI18_0]
-; CHECK-LE-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-LE-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-LE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-LE-NEXT: umov w8, v1.h[0]
-; CHECK-LE-NEXT: umov w9, v0.h[0]
-; CHECK-LE-NEXT: bfi w9, w8, #16, #16
-; CHECK-LE-NEXT: rbit w8, w9
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-LE-NEXT: mov x8, v0.d[1]
+; CHECK-LE-NEXT: fmov x9, d0
+; CHECK-LE-NEXT: rbit x10, x9
+; CHECK-LE-NEXT: cmp x9, #0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x10, x10
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: add x8, x8, #64
+; CHECK-LE-NEXT: csel x8, x10, x8, ne
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v32i8:
; CHECK-BE: // %bb.0:
-; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI18_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI18_0
-; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
; CHECK-BE-NEXT: cmlt v1.16b, v1.16b, #0
-; CHECK-BE-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-BE-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-BE-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-BE-NEXT: addp v1.16b, v1.16b, v1.16b
-; CHECK-BE-NEXT: rev16 v0.16b, v0.16b
-; CHECK-BE-NEXT: rev16 v1.16b, v1.16b
-; CHECK-BE-NEXT: umov w8, v0.h[0]
-; CHECK-BE-NEXT: umov w9, v1.h[0]
-; CHECK-BE-NEXT: bfi w9, w8, #16, #16
-; CHECK-BE-NEXT: rbit w8, w9
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: mov x8, v0.d[1]
+; CHECK-BE-NEXT: fmov x9, d0
+; CHECK-BE-NEXT: clz x10, x9
+; CHECK-BE-NEXT: cmp x9, #0
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: add x8, x8, #64
+; CHECK-BE-NEXT: csel x8, x10, x8, ne
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <32 x i8> %v, splat (i8 0)
%bm = bitcast <32 x i1> %cmp to i32
@@ -845,30 +703,23 @@ define i64 @cttz_v64i8_too_many_lanes(<64 x i8> %v) {
define i4 @cttz_v4i16(<4 x i16> %v) {
; CHECK-LE-LABEL: cttz_v4i16:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI20_0
; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
-; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI20_0]
-; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-LE-NEXT: addv h0, v0.4h
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x10
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v4i16:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
-; CHECK-BE-NEXT: adrp x8, .LCPI20_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_0
-; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
-; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-BE-NEXT: addv h0, v0.4h
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x10
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <4 x i16> %v, splat (i16 0)
%bm = bitcast <4 x i1> %cmp to i4
@@ -879,31 +730,23 @@ define i4 @cttz_v4i16(<4 x i16> %v) {
define i2 @cttz_v2i32(<2 x i32> %v) {
; CHECK-LE-LABEL: cttz_v2i32:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: mov x8, #1 // =0x1
; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
-; CHECK-LE-NEXT: movk x8, #2, lsl #32
-; CHECK-LE-NEXT: fmov d1, x8
-; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-LE-NEXT: addp v0.2s, v0.2s, v0.2s
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x4
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v2i32:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-NEXT: adrp x8, .LCPI21_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_0
-; CHECK-BE-NEXT: ld1 { v1.2s }, [x8]
; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
-; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-BE-NEXT: addp v0.2s, v0.2s, v0.2s
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x4
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <2 x i32> %v, splat (i32 0)
%bm = bitcast <2 x i1> %cmp to i2
@@ -915,32 +758,26 @@ define i2 @cttz_v2i8(<2 x i8> %v) {
; CHECK-LE-LABEL: cttz_v2i8:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: shl v0.2s, v0.2s, #24
-; CHECK-LE-NEXT: mov x8, #1 // =0x1
-; CHECK-LE-NEXT: movk x8, #2, lsl #32
-; CHECK-LE-NEXT: fmov d1, x8
+; CHECK-LE-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-LE-NEXT: cmlt v0.2s, v0.2s, #0
-; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-LE-NEXT: addp v0.2s, v0.2s, v0.2s
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x4
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v2i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.2s, v0.2s
-; CHECK-BE-NEXT: adrp x8, .LCPI22_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
-; CHECK-BE-NEXT: ld1 { v1.2s }, [x8]
; CHECK-BE-NEXT: shl v0.2s, v0.2s, #24
+; CHECK-BE-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-BE-NEXT: cmlt v0.2s, v0.2s, #0
-; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-BE-NEXT: addp v0.2s, v0.2s, v0.2s
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x4
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <2 x i8> %v, splat (i8 0)
%bm = bitcast <2 x i1> %cmp to i2
@@ -952,31 +789,26 @@ define i4 @cttz_v4i8(<4 x i8> %v) {
; CHECK-LE-LABEL: cttz_v4i8:
; CHECK-LE: // %bb.0:
; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-LE-NEXT: adrp x8, .LCPI23_0
-; CHECK-LE-NEXT: ldr d1, [x8, :lo12:.LCPI23_0]
+; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
-; CHECK-LE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-LE-NEXT: addv h0, v0.4h
-; CHECK-LE-NEXT: fmov w8, s0
-; CHECK-LE-NEXT: orr w8, w8, #0x10
-; CHECK-LE-NEXT: rbit w8, w8
-; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: cttz_v4i8:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.4h, v0.4h
-; CHECK-BE-NEXT: adrp x8, .LCPI23_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI23_0
-; CHECK-BE-NEXT: ld1 { v1.4h }, [x8]
; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
-; CHECK-BE-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-BE-NEXT: addv h0, v0.4h
-; CHECK-BE-NEXT: fmov w8, s0
-; CHECK-BE-NEXT: orr w8, w8, #0x10
-; CHECK-BE-NEXT: rbit w8, w8
-; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-BE-NEXT: ret
%cmp = icmp slt <4 x i8> %v, splat (i8 0)
%bm = bitcast <4 x i1> %cmp to i4
>From ad91b8c8f8e1bb3ed6f435f757c5c8db84043aeb Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 29 Jun 2026 00:44:53 +0000
Subject: [PATCH 3/4] [AArch64] Restructure narrowing loop + add v32i16/v16i32
tests
---
.../Target/AArch64/AArch64ISelLowering.cpp | 47 +++---
.../AArch64/cttz-of-bool-vector-bitcast.ll | 144 ++++++++++++++++--
2 files changed, 155 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d16bb609fec81..47bd7e7590305 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26803,35 +26803,32 @@ static SDValue performCTTZCombine(SDNode *N,
return DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Ctz);
};
- // Narrow toward CompressedBits. shrn-pair is only used once since it
- // leaves the i8 lanes already packed.
SDValue Cur = Mask;
- bool UsedShrnPair = false;
- while (Cur.getValueType().getSizeInBits() > CompressedBits) {
+ // Narrow wide lanes toward CompressedBits with xtn.
+ while (Cur.getValueType().getScalarSizeInBits() > MinLaneBits &&
+ Cur.getValueType().getSizeInBits() > CompressedBits) {
EVT CurVT = Cur.getValueType();
unsigned CurLaneBits = CurVT.getScalarSizeInBits();
- if (CurLaneBits == MinLaneBits) {
- if (UsedShrnPair)
- break;
- if (CurVT.getSizeInBits() <= QRegBits) {
- Cur = ShrnPair(Cur);
- } else {
- // Split into Q-reg-sized halves so each ShrnPair stays in range.
- EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx);
- SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
- DAG.getConstant(0, DL, MVT::i64));
- SDValue Hi = DAG.getNode(
- ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
- DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64));
- Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo),
- ShrnPair(Hi));
- }
- UsedShrnPair = true;
+ EVT NarrowVT = CurVT.changeVectorElementType(
+ Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2));
+ Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur);
+ }
+
+ // If still too wide, pack two lanes per byte with shrn.
+ if (Cur.getValueType().getSizeInBits() > CompressedBits) {
+ EVT CurVT = Cur.getValueType();
+ if (CurVT.getSizeInBits() <= QRegBits) {
+ Cur = ShrnPair(Cur);
} else {
- // For wider lanes, xtn each lane down to half its width.
- EVT NarrowVT = CurVT.changeVectorElementType(
- Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2));
- Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur);
+ // Split into Q-reg-sized halves so each ShrnPair stays in range.
+ EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx);
+ SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(0, DL, MVT::i64));
+ SDValue Hi = DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur,
+ DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64));
+ Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo),
+ ShrnPair(Hi));
}
}
diff --git a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
index dfded15396df7..f7186dbd2cc7c 100644
--- a/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/cttz-of-bool-vector-bitcast.ll
@@ -2,13 +2,6 @@
; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
-declare i16 @llvm.cttz.i16(i16, i1)
-declare i8 @llvm.cttz.i8(i8, i1)
-declare i4 @llvm.cttz.i4(i4, i1)
-declare i2 @llvm.cttz.i2(i2, i1)
-declare i32 @llvm.cttz.i32(i32, i1)
-declare i64 @llvm.cttz.i64(i64, i1)
-
define i16 @cttz_v16i8(<16 x i8> %v) {
; CHECK-LE-LABEL: cttz_v16i8:
; CHECK-LE: // %bb.0:
@@ -407,6 +400,28 @@ define i32 @cttz_bitcast_from_float(float %f) {
ret i32 %ctz
}
+define <4 x i32> @cttz_vector_result_no_fold(<4 x i32> %v) {
+; CHECK-LE-LABEL: cttz_vector_result_no_fold:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: rev32 v0.16b, v0.16b
+; CHECK-LE-NEXT: rbit v0.16b, v0.16b
+; CHECK-LE-NEXT: clz v0.4s, v0.4s
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_vector_result_no_fold:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: rev32 v0.16b, v0.16b
+; CHECK-BE-NEXT: rbit v0.16b, v0.16b
+; CHECK-BE-NEXT: clz v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ret
+ %ctz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %v, i1 false)
+ ret <4 x i32> %ctz
+}
+
define i3 @cttz_v3i8_non_pow2_lanes(<3 x i8> %v) {
; CHECK-LE-LABEL: cttz_v3i8_non_pow2_lanes:
; CHECK-LE: // %bb.0:
@@ -611,15 +626,122 @@ define i32 @cttz_v32i8(<32 x i8> %v) {
ret i32 %ctz
}
+define i32 @cttz_v32i16(<32 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v32i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: cmlt v3.8h, v3.8h, #0
+; CHECK-LE-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-LE-NEXT: mov x8, v0.d[1]
+; CHECK-LE-NEXT: fmov x9, d0
+; CHECK-LE-NEXT: rbit x10, x9
+; CHECK-LE-NEXT: cmp x9, #0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x10, x10
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: add x8, x8, #64
+; CHECK-LE-NEXT: csel x8, x10, x8, ne
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v32i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v2.8h, v2.8h
+; CHECK-BE-NEXT: rev64 v3.8h, v3.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-BE-NEXT: cmlt v3.8h, v3.8h, #0
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: shrn2 v0.16b, v1.8h, #4
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: mov x8, v0.d[1]
+; CHECK-BE-NEXT: fmov x9, d0
+; CHECK-BE-NEXT: clz x10, x9
+; CHECK-BE-NEXT: cmp x9, #0
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: add x8, x8, #64
+; CHECK-BE-NEXT: csel x8, x10, x8, ne
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <32 x i16> %v, splat (i16 0)
+ %bm = bitcast <32 x i1> %cmp to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %bm, i1 false)
+ ret i32 %ctz
+}
+
+define i16 @cttz_v16i32(<16 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v16i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-LE-NEXT: cmlt v2.4s, v2.4s, #0
+; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v3.4s, v3.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v2.4s, v2.4s
+; CHECK-BE-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8
+; CHECK-BE-NEXT: cmlt v3.4s, v3.4s, #0
+; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: cmlt v2.4s, v2.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i32> %v, splat (i32 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
define i64 @cttz_v64i8_too_many_lanes(<64 x i8> %v) {
; CHECK-LE-LABEL: cttz_v64i8_too_many_lanes:
; CHECK-LE: // %bb.0:
-; CHECK-LE-NEXT: adrp x8, .LCPI19_0
+; CHECK-LE-NEXT: adrp x8, .LCPI22_0
; CHECK-LE-NEXT: cmlt v3.16b, v3.16b, #0
; CHECK-LE-NEXT: cmlt v2.16b, v2.16b, #0
; CHECK-LE-NEXT: cmlt v1.16b, v1.16b, #0
; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-LE-NEXT: ldr q4, [x8, :lo12:.LCPI19_0]
+; CHECK-LE-NEXT: ldr q4, [x8, :lo12:.LCPI22_0]
; CHECK-LE-NEXT: and v3.16b, v3.16b, v4.16b
; CHECK-LE-NEXT: and v2.16b, v2.16b, v4.16b
; CHECK-LE-NEXT: and v1.16b, v1.16b, v4.16b
@@ -651,8 +773,8 @@ define i64 @cttz_v64i8_too_many_lanes(<64 x i8> %v) {
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
; CHECK-BE-NEXT: rev64 v3.16b, v3.16b
-; CHECK-BE-NEXT: adrp x8, .LCPI19_0
-; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI19_0
+; CHECK-BE-NEXT: adrp x8, .LCPI22_0
+; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI22_0
; CHECK-BE-NEXT: rev64 v2.16b, v2.16b
; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]
>From ea8c8a5f30fd3f0a49561c92219a28e55c352fd1 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 1 Jul 2026 22:58:13 +0000
Subject: [PATCH 4/4] [AArch64] Assert mask width after narrowing in
performCTTZCombine
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 47bd7e7590305..cb48273acb0e4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26832,6 +26832,10 @@ static SDValue performCTTZCombine(SDNode *N,
}
}
+ // Cur has been narrowed to CompressedBits.
+ assert(Cur.getValueType().getSizeInBits() == CompressedBits &&
+ "Unexpected mask width!");
+
SDValue Ctz = CompressedCttz(Cur);
unsigned LaneIndexShift = Log2_32(CompressedBits / NumLanes);
if (LaneIndexShift)
More information about the llvm-commits
mailing list