[llvm] [AArch64] Lower cttz(bitcast <Nxi1> to iN) with shrn-based compressed movemask (PR #199081)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 28 18:56:25 PDT 2026
================
@@ -0,0 +1,818 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i4 @llvm.cttz.i4(i4, i1)
+declare i2 @llvm.cttz.i2(i2, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
+define i16 @cttz_v16i8(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i16(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i8 @cttz_v8i8(<8 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v8i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8b, v0.8b
+; CHECK-BE-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i8> %v, splat (i8 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i16 @cttz_v16i8_with_any_check(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %any = icmp ne i16 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+notfound:
+ ret i16 16
+}
+
+
+define i4 @cttz_v4i32(<4 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i32> %v, splat (i32 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i2 @cttz_v2i64(<2 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v2i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: xtn v0.2s, v0.2d
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #5
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v2i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: xtn v0.2s, v0.2d
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #5
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <2 x i64> %v, splat (i64 0)
+ %bm = bitcast <2 x i1> %cmp to i2
+ %ctz = call i2 @llvm.cttz.i2(i2 %bm, i1 false)
+ ret i2 %ctz
+}
+
+
+define i16 @cttz_v16i8_eq(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_eq:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #42
+; CHECK-LE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_eq:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #42
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmeq v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp eq <16 x i8> %v, splat (i8 42)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_ne(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_ne:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_ne:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmtst v0.16b, v0.16b, v0.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp ne <16 x i8> %v, zeroinitializer
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i16 @cttz_v16i8_sgt(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_sgt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: movi v1.16b, #5
+; CHECK-LE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_sgt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: movi v1.16b, #5
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmgt v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp sgt <16 x i8> %v, splat (i8 5)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+
+define i4 @cttz_v4f32_olt(<4 x float> %v) {
+; CHECK-LE-LABEL: cttz_v4f32_olt:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4f32_olt:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: fcmlt v0.4s, v0.4s, #0.0
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = fcmp olt <4 x float> %v, zeroinitializer
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+
+define i8 @cttz_v8i16_with_any_check(<8 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-LE: // %bb.0: // %common.ret
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i16_with_any_check:
+; CHECK-BE: // %bb.0: // %common.ret
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i16> %v, splat (i16 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %any = icmp ne i8 %bm, 0
+ br i1 %any, label %found, label %notfound
+found:
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 true)
+ ret i8 %ctz
+notfound:
+ ret i8 8
+}
+
+
+define i16 @cttz_v16i8_zero_undef(<16 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i8_zero_undef:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i8> %v, splat (i8 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 true)
+ ret i16 %ctz
+}
+
+
+define i32 @cttz_plain_i32(i32 %x) {
+; CHECK-LABEL: cttz_plain_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rbit w8, w0
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %ctz = call i32 @llvm.cttz.i32(i32 %x, i1 false)
+ ret i32 %ctz
+}
+
+define i32 @cttz_bitcast_from_float(float %f) {
+; CHECK-LABEL: cttz_bitcast_from_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: rbit w8, w8
+; CHECK-NEXT: clz w0, w8
+; CHECK-NEXT: ret
+ %i = bitcast float %f to i32
+ %ctz = call i32 @llvm.cttz.i32(i32 %i, i1 false)
+ ret i32 %ctz
+}
+
+define i3 @cttz_v3i8_non_pow2_lanes(<3 x i8> %v) {
+; CHECK-LE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sub sp, sp, #16
+; CHECK-LE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-LE-NEXT: fmov s0, w0
+; CHECK-LE-NEXT: mov v0.h[1], w1
+; CHECK-LE-NEXT: mov v0.h[2], w2
+; CHECK-LE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-LE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-LE-NEXT: umov w8, v0.h[0]
+; CHECK-LE-NEXT: umov w9, v0.h[1]
+; CHECK-LE-NEXT: umov w10, v0.h[2]
+; CHECK-LE-NEXT: and w8, w8, #0x1
+; CHECK-LE-NEXT: bfi w8, w9, #1, #1
+; CHECK-LE-NEXT: bfi w8, w10, #2, #1
+; CHECK-LE-NEXT: orr w8, w8, #0x8
+; CHECK-LE-NEXT: rbit w8, w8
+; CHECK-LE-NEXT: clz w0, w8
+; CHECK-LE-NEXT: add sp, sp, #16
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v3i8_non_pow2_lanes:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: sub sp, sp, #16
+; CHECK-BE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-BE-NEXT: fmov s0, w0
+; CHECK-BE-NEXT: mov v0.h[1], w1
+; CHECK-BE-NEXT: mov v0.h[2], w2
+; CHECK-BE-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-BE-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-BE-NEXT: umov w8, v0.h[1]
+; CHECK-BE-NEXT: umov w9, v0.h[0]
+; CHECK-BE-NEXT: umov w10, v0.h[2]
+; CHECK-BE-NEXT: ubfiz w8, w8, #1, #1
+; CHECK-BE-NEXT: bfi w8, w9, #2, #1
+; CHECK-BE-NEXT: bfxil w8, w10, #0, #1
+; CHECK-BE-NEXT: orr w8, w8, #0x8
+; CHECK-BE-NEXT: rbit w8, w8
+; CHECK-BE-NEXT: clz w0, w8
+; CHECK-BE-NEXT: add sp, sp, #16
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <3 x i8> %v, splat (i8 0)
+ %bm = bitcast <3 x i1> %cmp to i3
+ %ctz = call i3 @llvm.cttz.i3(i3 %bm, i1 false)
+ ret i3 %ctz
+}
+declare i3 @llvm.cttz.i3(i3, i1)
+
+define i16 @cttz_v16i16(<16 x i16> %v) {
+; CHECK-LE-LABEL: cttz_v16i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-LE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-LE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-LE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #2
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v16i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.8h, v1.8h, #0
+; CHECK-BE-NEXT: cmlt v0.8h, v0.8h, #0
+; CHECK-BE-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT: shrn v0.8b, v0.8h, #4
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #2
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <16 x i16> %v, splat (i16 0)
+ %bm = bitcast <16 x i1> %cmp to i16
+ %ctz = call i16 @llvm.cttz.i16(i16 %bm, i1 false)
+ ret i16 %ctz
+}
+
+define i8 @cttz_v8i32(<8 x i32> %v) {
+; CHECK-LE-LABEL: cttz_v8i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-LE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-LE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-LE-NEXT: xtn v0.8b, v0.8h
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #3
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v8i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.4s, v1.4s, #0
+; CHECK-BE-NEXT: cmlt v0.4s, v0.4s, #0
+; CHECK-BE-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BE-NEXT: xtn v0.8b, v0.8h
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #3
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <8 x i32> %v, splat (i32 0)
+ %bm = bitcast <8 x i1> %cmp to i8
+ %ctz = call i8 @llvm.cttz.i8(i8 %bm, i1 false)
+ ret i8 %ctz
+}
+
+define i4 @cttz_v4i64(<4 x i64> %v) {
+; CHECK-LE-LABEL: cttz_v4i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-LE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-LE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-LE-NEXT: xtn v0.4h, v0.4s
+; CHECK-LE-NEXT: fmov x8, d0
+; CHECK-LE-NEXT: rbit x8, x8
+; CHECK-LE-NEXT: clz x8, x8
+; CHECK-LE-NEXT: lsr x0, x8, #4
+; CHECK-LE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: cttz_v4i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: cmlt v1.2d, v1.2d, #0
+; CHECK-BE-NEXT: cmlt v0.2d, v0.2d, #0
+; CHECK-BE-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-BE-NEXT: xtn v0.4h, v0.4s
+; CHECK-BE-NEXT: fmov x8, d0
+; CHECK-BE-NEXT: rbit x8, x8
+; CHECK-BE-NEXT: clz x8, x8
+; CHECK-BE-NEXT: lsr x0, x8, #4
+; CHECK-BE-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-BE-NEXT: ret
+ %cmp = icmp slt <4 x i64> %v, splat (i64 0)
+ %bm = bitcast <4 x i1> %cmp to i4
+ %ctz = call i4 @llvm.cttz.i4(i4 %bm, i1 false)
+ ret i4 %ctz
+}
+
+define i32 @cttz_v32i8(<32 x i8> %v) {
----------------
as4230 wrote:
Yep, added tests for v32i16 and v16i32. I also pulled the original tests into a precommit commit so you can see the before and after codegen as a clean diff while reviewing.
https://github.com/llvm/llvm-project/pull/199081
More information about the llvm-commits
mailing list