[llvm] [AArch64] Improve fixed vector lowering for cttz/ctlz when sve (PR #192427)
Yashwant Singh via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 00:21:43 PDT 2026
https://github.com/yashssh updated https://github.com/llvm/llvm-project/pull/192427
>From ba6579714e5d374c5061f687df09a67b1e3774b7 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 04:34:35 -0700
Subject: [PATCH 1/5] [NFC]Pre-commit tests
---
.../AArch64/ctlz-cttz-fixed-vectors.ll | 386 ++++++++++++++++++
1 file changed, 386 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
new file mode 100644
index 0000000000000..a75a6a06e994f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -0,0 +1,386 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=NEON
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s --check-prefix=SVE
+
+; ============================================================
+; CTLZ
+; ============================================================
+
+; --- 64-bit vectors ---
+
+define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
+; NEON-LABEL: ctlz_v8i8:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.8b, v0.8b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v8i8:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.8b, v0.8b
+; SVE-NEXT: ret
+ %r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
+; NEON-LABEL: ctlz_v4i16:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.4h, v0.4h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v4i16:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: ret
+ %r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
+; NEON-LABEL: ctlz_v2i32:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.2s, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v2i32:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: ret
+ %r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @ctlz_v1i64(<1 x i64> %a) {
+; NEON-LABEL: ctlz_v1i64:
+; NEON: // %bb.0:
+; NEON-NEXT: ushr d1, d0, #1
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #2
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #4
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #8
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #16
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #32
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: mvn v0.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: uaddlp v0.1d, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v1i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ushr d1, d0, #1
+; SVE-NEXT: ptrue p0.d, vl1
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #2
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #4
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #8
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #16
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #32
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: mvn v0.8b, v0.8b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %r = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false)
+ ret <1 x i64> %r
+}
+
+; --- 128-bit vectors ---
+
+define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
+; NEON-LABEL: ctlz_v16i8:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.16b, v0.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.16b, v0.16b
+; SVE-NEXT: ret
+ %r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
+; NEON-LABEL: ctlz_v8i16:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.8h, v0.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: ret
+ %r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
+; NEON-LABEL: ctlz_v4i32:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.4s, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: ret
+ %r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
+; NEON-LABEL: ctlz_v2i64:
+; NEON: // %bb.0:
+; NEON-NEXT: ushr v1.2d, v0.2d, #1
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #2
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #4
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #8
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #16
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #32
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: mvn v0.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ushr v1.2d, v0.2d, #1
+; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #2
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #4
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #8
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #16
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #32
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: mvn v0.16b, v0.16b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: ret
+ %r = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
+ ret <2 x i64> %r
+}
+
+; ============================================================
+; CTTZ
+; ============================================================
+
+; --- 64-bit vectors ---
+
+define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
+; NEON-LABEL: cttz_v8i8:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.8b, #1
+; NEON-NEXT: sub v1.8b, v0.8b, v1.8b
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v8i8:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.8b, #1
+; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: ret
+ %r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
+; NEON-LABEL: cttz_v4i16:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.4h, #1
+; NEON-NEXT: sub v1.4h, v0.4h, v1.4h
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: movi v1.4h, #16
+; NEON-NEXT: clz v0.4h, v0.4h
+; NEON-NEXT: sub v0.4h, v1.4h, v0.4h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v4i16:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.4h, #1
+; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
+; NEON-LABEL: cttz_v2i32:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.2s, #1
+; NEON-NEXT: sub v1.2s, v0.2s, v1.2s
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: movi v1.2s, #32
+; NEON-NEXT: clz v0.2s, v0.2s
+; NEON-NEXT: sub v0.2s, v1.2s, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v2i32:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.2s, #1
+; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; SVE-NEXT: ret
+ %r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
+; NEON-LABEL: cttz_v1i64:
+; NEON: // %bb.0:
+; NEON-NEXT: mov w8, #1 // =0x1
+; NEON-NEXT: fmov d1, x8
+; NEON-NEXT: sub d1, d0, d1
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: uaddlp v0.1d, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v1i64:
+; SVE: // %bb.0:
+; SVE-NEXT: mov w8, #1 // =0x1
+; SVE-NEXT: ptrue p0.d, vl1
+; SVE-NEXT: fmov d1, x8
+; SVE-NEXT: sub d1, d0, d1
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %r = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 false)
+ ret <1 x i64> %r
+}
+
+; --- 128-bit vectors ---
+
+define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
+; NEON-LABEL: cttz_v16i8:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.16b, #1
+; NEON-NEXT: sub v1.16b, v0.16b, v1.16b
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: ret
+ %r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
+; NEON-LABEL: cttz_v8i16:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.8h, #1
+; NEON-NEXT: sub v1.8h, v0.8h, v1.8h
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: movi v1.8h, #16
+; NEON-NEXT: clz v0.8h, v0.8h
+; NEON-NEXT: sub v0.8h, v1.8h, v0.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
+; SVE-NEXT: movi v1.8h, #16
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
+; SVE-NEXT: ret
+ %r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
+; NEON-LABEL: cttz_v4i32:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.4s, #1
+; NEON-NEXT: sub v1.4s, v0.4s, v1.4s
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: movi v1.4s, #32
+; NEON-NEXT: clz v0.4s, v0.4s
+; NEON-NEXT: sub v0.4s, v1.4s, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
+; SVE-NEXT: movi v1.4s, #32
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
+; SVE-NEXT: ret
+ %r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @cttz_v2i64(<2 x i64> %a) {
+; NEON-LABEL: cttz_v2i64:
+; NEON: // %bb.0:
+; NEON-NEXT: mov w8, #1 // =0x1
+; NEON-NEXT: dup v1.2d, x8
+; NEON-NEXT: sub v1.2d, v0.2d, v1.2d
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.d, z1.d, #1 // =0x1
+; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: ret
+ %r = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+ ret <2 x i64> %r
+}
>From 056739f91e1e9397fb974c6ea3bc3473c930eeed Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 04:36:20 -0700
Subject: [PATCH 2/5] [AArch64] Improve fixed vector lowering for cttz/ctlz
when sve
When available we should be able to lower fixed size 64/128 bit cttz/ctlz vector instructions using sve. Below are the performance uplifts
+--------------+--------------+--------------+
| Function | CTTZ Speedup | CTLZ Speedup |
+--------------+--------------+--------------+
| u8x8 | 1.33x | 1.00x |
| u16x4 | 1.26x | 1.00x |
| u32x2 | 1.26x | 1.00x |
| u64x1 | 1.00x | 7.00x |
+--------------+--------------+--------------+
| u8x16 | 1.67x | 1.00x |
| u16x8 | 1.51x | 1.00x |
| u32x4 | 1.51x | 1.00x |
| u64x2 | 1.26x | 7.00x |
+--------------+--------------+--------------+
---
.../Target/AArch64/AArch64ISelLowering.cpp | 12 +-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 141 ++++++++----------
2 files changed, 69 insertions(+), 84 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4b6b0758006af..9cae72cedaae5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1930,6 +1930,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
+ MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ }
+
// NOTE: Currently this has to happen after computeRegisterProperties rather
// than the preferred option of combining it with the addRegisterClass call.
if (Subtarget->useSVEForFixedLengthVectors()) {
@@ -1957,9 +1963,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// These operations are not supported on NEON but SVE can do them.
setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
- setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
@@ -1977,7 +1980,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::CTTZ, VT, Custom);
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
@@ -12070,7 +12072,7 @@ SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
assert(VT.isScalableVector() ||
useSVEForFixedLengthVectorVT(
- VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
+ VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()));
SDLoc DL(Op);
SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index a75a6a06e994f..2032434880995 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -16,7 +16,10 @@ define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: ctlz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.b, vl8
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -30,7 +33,10 @@ define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: ctlz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -44,7 +50,10 @@ define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: ctlz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: ptrue p0.s, vl2
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -74,21 +83,9 @@ define <1 x i64> @ctlz_v1i64(<1 x i64> %a) {
;
; SVE-LABEL: ctlz_v1i64:
; SVE: // %bb.0:
-; SVE-NEXT: ushr d1, d0, #1
; SVE-NEXT: ptrue p0.d, vl1
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #2
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #4
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #8
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #16
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #32
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: mvn v0.8b, v0.8b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false)
@@ -105,7 +102,10 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: ctlz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.b, vl16
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -119,7 +119,10 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: ctlz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -133,7 +136,10 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: ctlz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -163,21 +169,9 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
;
; SVE-LABEL: ctlz_v2i64:
; SVE: // %bb.0:
-; SVE-NEXT: ushr v1.2d, v0.2d, #1
; SVE-NEXT: ptrue p0.d, vl2
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #2
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #4
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #8
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #16
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #32
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: mvn v0.16b, v0.16b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
@@ -201,10 +195,10 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.8b, #1
-; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.b, vl8
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -223,12 +217,11 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.4h, #1
-; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.4h, #16
-; SVE-NEXT: clz v0.4h, v0.4h
-; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; SVE-NEXT: rev16 v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -247,12 +240,11 @@ define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.2s, #1
-; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.2s, #32
-; SVE-NEXT: clz v0.2s, v0.2s
-; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; SVE-NEXT: rev32 v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.s, vl2
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -273,12 +265,10 @@ define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
;
; SVE-LABEL: cttz_v1i64:
; SVE: // %bb.0:
-; SVE-NEXT: mov w8, #1 // =0x1
+; SVE-NEXT: rev64 v0.8b, v0.8b
; SVE-NEXT: ptrue p0.d, vl1
-; SVE-NEXT: fmov d1, x8
-; SVE-NEXT: sub d1, d0, d1
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 false)
@@ -298,11 +288,10 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.b, vl16
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -321,13 +310,11 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
-; SVE-NEXT: movi v1.8h, #16
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
-; SVE-NEXT: clz v0.8h, v0.8h
-; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
+; SVE-NEXT: rev16 v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -346,13 +333,11 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
-; SVE-NEXT: movi v1.4s, #32
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
-; SVE-NEXT: clz v0.4s, v0.4s
-; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
+; SVE-NEXT: rev32 v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -373,12 +358,10 @@ define <2 x i64> @cttz_v2i64(<2 x i64> %a) {
;
; SVE-LABEL: cttz_v2i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.d, z1.d, #1 // =0x1
+; SVE-NEXT: rev64 v0.16b, v0.16b
; SVE-NEXT: ptrue p0.d, vl2
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
>From f2ab8b85e14332badc02a9db1cf0686b6b7a5962 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 05:22:28 -0700
Subject: [PATCH 3/5] clang-format
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9cae72cedaae5..0a8d8d3cce26c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1930,10 +1930,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
- MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::CTTZ, VT, Custom);
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
}
// NOTE: Currently this has to happen after computeRegisterProperties rather
>From c8d419dee86cebd069cb7f42472c13795f9fafdd Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 21:30:09 -0700
Subject: [PATCH 4/5] Only lower vNi64 with sve
---
.../Target/AArch64/AArch64ISelLowering.cpp | 7 +-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 93 ++++++++-----------
2 files changed, 46 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0a8d8d3cce26c..77c15cf662f4d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1930,8 +1930,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
- MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ for (auto VT : {MVT::v1i64, MVT::v2i64}) {
setOperationAction(ISD::CTLZ, VT, Custom);
setOperationAction(ISD::CTTZ, VT, Custom);
}
@@ -1984,6 +1983,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
}
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
+ MVT::v2i32, MVT::v4i32}) {
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ }
// Use SVE for vectors with more than 2 elements.
for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index 2032434880995..92da881f2b3bf 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -16,10 +16,7 @@ define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: ctlz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.b, vl8
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -33,10 +30,7 @@ define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: ctlz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.h, vl4
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -50,10 +44,7 @@ define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: ctlz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s, vl2
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -102,10 +93,7 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: ctlz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.b, vl16
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -119,10 +107,7 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: ctlz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.h, vl8
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -136,10 +121,7 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: ctlz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -195,10 +177,10 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.b, vl8
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.8b, #1
+; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -217,11 +199,12 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: rev16 v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.h, vl4
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.4h, #1
+; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -240,11 +223,12 @@ define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: rev32 v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.s, vl2
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.2s, #1
+; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -288,10 +272,11 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.b, vl16
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -310,11 +295,13 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: rev16 v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.h, vl8
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
+; SVE-NEXT: movi v1.8h, #16
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -333,11 +320,13 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: rev32 v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
+; SVE-NEXT: movi v1.4s, #32
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
>From 99417670b751ee2446f960d91a766104c4e2c902 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 22:10:22 -0700
Subject: [PATCH 5/5] Custom lowering path for fixedlen_vector_valuetypes cttz
---
.../Target/AArch64/AArch64ISelLowering.cpp | 13 +-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 83 ++++--------
llvm/test/CodeGen/AArch64/cttz.ll | 119 ++++++------------
3 files changed, 70 insertions(+), 145 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 77c15cf662f4d..b496213605885 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1422,7 +1422,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BSWAP, VT, Legal);
else
setOperationAction(ISD::BSWAP, VT, Expand);
- setOperationAction(ISD::CTTZ, VT, Expand);
+ if (VT == MVT::v8i8 || VT == MVT::v16i8 || VT == MVT::v4i16 ||
+ VT == MVT::v8i16 || VT == MVT::v2i32 || VT == MVT::v4i32)
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ else
+ setOperationAction(ISD::CTTZ, VT, Expand);
for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
setTruncStoreAction(VT, InnerVT, Expand);
@@ -1983,8 +1987,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
- MVT::v2i32, MVT::v4i32}) {
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32}) {
setOperationAction(ISD::CTTZ, VT, Custom);
}
@@ -12075,7 +12079,8 @@ SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
assert(VT.isScalableVector() ||
useSVEForFixedLengthVectorVT(
- VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()));
+ VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()) ||
+ VT.getScalarSizeInBits() < 64);
SDLoc DL(Op);
SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index 92da881f2b3bf..ba313e4fb2fdc 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -169,18 +169,14 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
; NEON-LABEL: cttz_v8i8:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.8b, #1
-; NEON-NEXT: sub v1.8b, v0.8b, v1.8b
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
+; NEON-NEXT: clz v0.8b, v0.8b
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.8b, #1
-; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -189,22 +185,16 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
; NEON-LABEL: cttz_v4i16:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.4h, #1
-; NEON-NEXT: sub v1.4h, v0.4h, v1.4h
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: movi v1.4h, #16
+; NEON-NEXT: rev16 v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
; NEON-NEXT: clz v0.4h, v0.4h
-; NEON-NEXT: sub v0.4h, v1.4h, v0.4h
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.4h, #1
-; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: rev16 v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
; SVE-NEXT: clz v0.4h, v0.4h
-; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -213,22 +203,16 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
; NEON-LABEL: cttz_v2i32:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.2s, #1
-; NEON-NEXT: sub v1.2s, v0.2s, v1.2s
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: movi v1.2s, #32
+; NEON-NEXT: rev32 v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
; NEON-NEXT: clz v0.2s, v0.2s
-; NEON-NEXT: sub v0.2s, v1.2s, v0.2s
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.2s, #1
-; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: rev32 v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
; SVE-NEXT: clz v0.2s, v0.2s
-; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -264,19 +248,14 @@ define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
; NEON-LABEL: cttz_v16i8:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.16b, #1
-; NEON-NEXT: sub v1.16b, v0.16b, v1.16b
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
+; NEON-NEXT: clz v0.16b, v0.16b
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -285,23 +264,16 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
; NEON-LABEL: cttz_v8i16:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.8h, #1
-; NEON-NEXT: sub v1.8h, v0.8h, v1.8h
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: movi v1.8h, #16
+; NEON-NEXT: rev16 v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
; NEON-NEXT: clz v0.8h, v0.8h
-; NEON-NEXT: sub v0.8h, v1.8h, v0.8h
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
-; SVE-NEXT: movi v1.8h, #16
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: rev16 v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
; SVE-NEXT: clz v0.8h, v0.8h
-; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -310,23 +282,16 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
; NEON-LABEL: cttz_v4i32:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.4s, #1
-; NEON-NEXT: sub v1.4s, v0.4s, v1.4s
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: movi v1.4s, #32
+; NEON-NEXT: rev32 v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
; NEON-NEXT: clz v0.4s, v0.4s
-; NEON-NEXT: sub v0.4s, v1.4s, v0.4s
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
-; SVE-NEXT: movi v1.4s, #32
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: rev32 v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
; SVE-NEXT: clz v0.4s, v0.4s
-; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index c9181b4c312d1..86c1aca9ed9e1 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -7,15 +7,12 @@ define void @v2i8(ptr %p1) {
; CHECK-SD-LABEL: v2i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr h0, [x0]
-; CHECK-SD-NEXT: movi v1.2s, #1
; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: orr v0.2s, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: mov s1, v0.s[1]
; CHECK-SD-NEXT: str b0, [x0]
; CHECK-SD-NEXT: stur b1, [x0, #1]
@@ -49,14 +46,11 @@ define void @v3i8(ptr %p1) {
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.4h, #1
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: orr v0.4h, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
@@ -101,14 +95,11 @@ define void @v4i8(ptr %p1) {
; CHECK-SD-LABEL: v4i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.4h, #1
; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: orr v0.4h, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: str s0, [x0]
; CHECK-SD-NEXT: ret
@@ -149,10 +140,8 @@ entry:
define <8 x i8> @v8i8(<8 x i8> %d) {
; CHECK-SD-LABEL: v8i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.8b, #1
-; CHECK-SD-NEXT: sub v1.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: cnt v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
+; CHECK-SD-NEXT: clz v0.8b, v0.8b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i8:
@@ -170,10 +159,8 @@ entry:
define <16 x i8> @v16i8(<16 x i8> %d) {
; CHECK-SD-LABEL: v16i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.16b, #1
-; CHECK-SD-NEXT: sub v1.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v16i8:
@@ -191,13 +178,10 @@ entry:
define <32 x i8> @v32i8(<32 x i8> %d) {
; CHECK-SD-LABEL: v32i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.16b, #1
-; CHECK-SD-NEXT: sub v3.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: sub v2.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
-; CHECK-SD-NEXT: cnt v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v1.16b, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v32i8:
@@ -219,14 +203,11 @@ define void @v2i16(ptr %p1) {
; CHECK-SD-LABEL: v2i16:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.2s, #1
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: orr v0.2s, #1, lsl #16
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: mov s1, v0.s[1]
; CHECK-SD-NEXT: str h0, [x0]
; CHECK-SD-NEXT: str h1, [x0, #2]
@@ -260,13 +241,10 @@ entry:
define void @v3i16(ptr %p1) {
; CHECK-SD-LABEL: v3i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.4h, #1
-; CHECK-SD-NEXT: ldr d1, [x0]
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
-; CHECK-SD-NEXT: bic v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: ldr d0, [x0]
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: mov h1, v0.h[2]
; CHECK-SD-NEXT: str s0, [x0]
; CHECK-SD-NEXT: str h1, [x0, #4]
@@ -300,12 +278,9 @@ entry:
define <4 x i16> @v4i16(<4 x i16> %d) {
; CHECK-SD-LABEL: v4i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4h, #1
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v4i16:
@@ -324,12 +299,9 @@ entry:
define <8 x i16> @v8i16(<8 x i16> %d) {
; CHECK-SD-LABEL: v8i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.8h, #1
-; CHECK-SD-NEXT: sub v1.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.8h, #16
+; CHECK-SD-NEXT: rev16 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.8h, v0.8h
-; CHECK-SD-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i16:
@@ -348,16 +320,12 @@ entry:
define <16 x i16> @v16i16(<16 x i16> %d) {
; CHECK-SD-LABEL: v16i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.8h, #1
-; CHECK-SD-NEXT: sub v3.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT: sub v2.8h, v1.8h, v2.8h
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: movi v2.8h, #16
+; CHECK-SD-NEXT: rev16 v0.16b, v0.16b
+; CHECK-SD-NEXT: rev16 v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
; CHECK-SD-NEXT: clz v0.8h, v0.8h
; CHECK-SD-NEXT: clz v1.8h, v1.8h
-; CHECK-SD-NEXT: sub v0.8h, v2.8h, v0.8h
-; CHECK-SD-NEXT: sub v1.8h, v2.8h, v1.8h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v16i16:
@@ -380,12 +348,9 @@ entry:
define <2 x i32> @v2i32(<2 x i32> %d) {
; CHECK-SD-LABEL: v2i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2s, #1
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v2i32:
@@ -405,12 +370,9 @@ entry:
define <3 x i32> @v3i32(<3 x i32> %d) {
; CHECK-SD-LABEL: v3i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4s, #1
-; CHECK-SD-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
-; CHECK-SD-NEXT: sub v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v3i32:
@@ -434,12 +396,9 @@ entry:
define <4 x i32> @v4i32(<4 x i32> %d) {
; CHECK-SD-LABEL: v4i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4s, #1
-; CHECK-SD-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
-; CHECK-SD-NEXT: sub v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v4i32:
@@ -459,16 +418,12 @@ entry:
define <8 x i32> @v8i32(<8 x i32> %d) {
; CHECK-SD-LABEL: v8i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.4s, #1
-; CHECK-SD-NEXT: sub v3.4s, v0.4s, v2.4s
-; CHECK-SD-NEXT: sub v2.4s, v1.4s, v2.4s
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: movi v2.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rev32 v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
; CHECK-SD-NEXT: clz v1.4s, v1.4s
-; CHECK-SD-NEXT: sub v0.4s, v2.4s, v0.4s
-; CHECK-SD-NEXT: sub v1.4s, v2.4s, v1.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i32:
More information about the llvm-commits
mailing list