[llvm] [AArch64] Improve fixed vector lowering for cttz/ctlz when sve (PR #192427)
Yashwant Singh via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 01:08:28 PDT 2026
https://github.com/yashssh updated https://github.com/llvm/llvm-project/pull/192427
>From cf518c0c0caa19b435ddeedf1434ad8c18022180 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 04:34:35 -0700
Subject: [PATCH 1/5] [NFC]Pre-commit tests
---
.../AArch64/ctlz-cttz-fixed-vectors.ll | 386 ++++++++++++++++++
1 file changed, 386 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
new file mode 100644
index 0000000000000..a75a6a06e994f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -0,0 +1,386 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=NEON
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s --check-prefix=SVE
+
+; ============================================================
+; CTLZ
+; ============================================================
+
+; --- 64-bit vectors ---
+
+define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
+; NEON-LABEL: ctlz_v8i8:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.8b, v0.8b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v8i8:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.8b, v0.8b
+; SVE-NEXT: ret
+ %r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
+; NEON-LABEL: ctlz_v4i16:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.4h, v0.4h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v4i16:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: ret
+ %r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
+; NEON-LABEL: ctlz_v2i32:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.2s, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v2i32:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: ret
+ %r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @ctlz_v1i64(<1 x i64> %a) {
+; NEON-LABEL: ctlz_v1i64:
+; NEON: // %bb.0:
+; NEON-NEXT: ushr d1, d0, #1
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #2
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #4
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #8
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #16
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: ushr d1, d0, #32
+; NEON-NEXT: orr v0.8b, v0.8b, v1.8b
+; NEON-NEXT: mvn v0.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: uaddlp v0.1d, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v1i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ushr d1, d0, #1
+; SVE-NEXT: ptrue p0.d, vl1
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #2
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #4
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #8
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #16
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: ushr d1, d0, #32
+; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
+; SVE-NEXT: mvn v0.8b, v0.8b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %r = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false)
+ ret <1 x i64> %r
+}
+
+; --- 128-bit vectors ---
+
+define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
+; NEON-LABEL: ctlz_v16i8:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.16b, v0.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.16b, v0.16b
+; SVE-NEXT: ret
+ %r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
+; NEON-LABEL: ctlz_v8i16:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.8h, v0.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: ret
+ %r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
+; NEON-LABEL: ctlz_v4i32:
+; NEON: // %bb.0:
+; NEON-NEXT: clz v0.4s, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: ret
+ %r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
+; NEON-LABEL: ctlz_v2i64:
+; NEON: // %bb.0:
+; NEON-NEXT: ushr v1.2d, v0.2d, #1
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #2
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #4
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #8
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #16
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: ushr v1.2d, v0.2d, #32
+; NEON-NEXT: orr v0.16b, v0.16b, v1.16b
+; NEON-NEXT: mvn v0.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: ctlz_v2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: ushr v1.2d, v0.2d, #1
+; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #2
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #4
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #8
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #16
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: ushr v1.2d, v0.2d, #32
+; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
+; SVE-NEXT: mvn v0.16b, v0.16b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: ret
+ %r = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
+ ret <2 x i64> %r
+}
+
+; ============================================================
+; CTTZ
+; ============================================================
+
+; --- 64-bit vectors ---
+
+define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
+; NEON-LABEL: cttz_v8i8:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.8b, #1
+; NEON-NEXT: sub v1.8b, v0.8b, v1.8b
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v8i8:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.8b, #1
+; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: ret
+ %r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
+ ret <8 x i8> %r
+}
+
+define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
+; NEON-LABEL: cttz_v4i16:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.4h, #1
+; NEON-NEXT: sub v1.4h, v0.4h, v1.4h
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: movi v1.4h, #16
+; NEON-NEXT: clz v0.4h, v0.4h
+; NEON-NEXT: sub v0.4h, v1.4h, v0.4h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v4i16:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.4h, #1
+; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
+ ret <4 x i16> %r
+}
+
+define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
+; NEON-LABEL: cttz_v2i32:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.2s, #1
+; NEON-NEXT: sub v1.2s, v0.2s, v1.2s
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: movi v1.2s, #32
+; NEON-NEXT: clz v0.2s, v0.2s
+; NEON-NEXT: sub v0.2s, v1.2s, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v2i32:
+; SVE: // %bb.0:
+; SVE-NEXT: movi v1.2s, #1
+; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; SVE-NEXT: ret
+ %r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+ ret <2 x i32> %r
+}
+
+define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
+; NEON-LABEL: cttz_v1i64:
+; NEON: // %bb.0:
+; NEON-NEXT: mov w8, #1 // =0x1
+; NEON-NEXT: fmov d1, x8
+; NEON-NEXT: sub d1, d0, d1
+; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
+; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: uaddlp v0.4h, v0.8b
+; NEON-NEXT: uaddlp v0.2s, v0.4h
+; NEON-NEXT: uaddlp v0.1d, v0.2s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v1i64:
+; SVE: // %bb.0:
+; SVE-NEXT: mov w8, #1 // =0x1
+; SVE-NEXT: ptrue p0.d, vl1
+; SVE-NEXT: fmov d1, x8
+; SVE-NEXT: sub d1, d0, d1
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %r = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 false)
+ ret <1 x i64> %r
+}
+
+; --- 128-bit vectors ---
+
+define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
+; NEON-LABEL: cttz_v16i8:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.16b, #1
+; NEON-NEXT: sub v1.16b, v0.16b, v1.16b
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v16i8:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: ret
+ %r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+ ret <16 x i8> %r
+}
+
+define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
+; NEON-LABEL: cttz_v8i16:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.8h, #1
+; NEON-NEXT: sub v1.8h, v0.8h, v1.8h
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: movi v1.8h, #16
+; NEON-NEXT: clz v0.8h, v0.8h
+; NEON-NEXT: sub v0.8h, v1.8h, v0.8h
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v8i16:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
+; SVE-NEXT: movi v1.8h, #16
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
+; SVE-NEXT: ret
+ %r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+ ret <8 x i16> %r
+}
+
+define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
+; NEON-LABEL: cttz_v4i32:
+; NEON: // %bb.0:
+; NEON-NEXT: movi v1.4s, #1
+; NEON-NEXT: sub v1.4s, v0.4s, v1.4s
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: movi v1.4s, #32
+; NEON-NEXT: clz v0.4s, v0.4s
+; NEON-NEXT: sub v0.4s, v1.4s, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v4i32:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
+; SVE-NEXT: movi v1.4s, #32
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
+; SVE-NEXT: ret
+ %r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+ ret <4 x i32> %r
+}
+
+define <2 x i64> @cttz_v2i64(<2 x i64> %a) {
+; NEON-LABEL: cttz_v2i64:
+; NEON: // %bb.0:
+; NEON-NEXT: mov w8, #1 // =0x1
+; NEON-NEXT: dup v1.2d, x8
+; NEON-NEXT: sub v1.2d, v0.2d, v1.2d
+; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
+; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: uaddlp v0.8h, v0.16b
+; NEON-NEXT: uaddlp v0.4s, v0.8h
+; NEON-NEXT: uaddlp v0.2d, v0.4s
+; NEON-NEXT: ret
+;
+; SVE-LABEL: cttz_v2i64:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.d, z1.d, #1 // =0x1
+; SVE-NEXT: ptrue p0.d, vl2
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: ret
+ %r = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+ ret <2 x i64> %r
+}
>From f30d287931fa49e6778e4885f1158f02945c8370 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 04:36:20 -0700
Subject: [PATCH 2/5] [AArch64] Improve fixed vector lowering for cttz/ctlz
when sve
When available we should be able to lower fixed size 64/128 bit cttz/ctlz vector instructions using sve. Below are the performance uplifts
+--------------+--------------+--------------+
| Function | CTTZ Speedup | CTLZ Speedup |
+--------------+--------------+--------------+
| u8x8 | 1.33x | 1.00x |
| u16x4 | 1.26x | 1.00x |
| u32x2 | 1.26x | 1.00x |
| u64x1 | 1.00x | 7.00x |
+--------------+--------------+--------------+
| u8x16 | 1.67x | 1.00x |
| u16x8 | 1.51x | 1.00x |
| u32x4 | 1.51x | 1.00x |
| u64x2 | 1.26x | 7.00x |
+--------------+--------------+--------------+
---
.../Target/AArch64/AArch64ISelLowering.cpp | 12 +-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 141 ++++++++----------
2 files changed, 69 insertions(+), 84 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9b34d9b385b4e..a45ff186a658c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1935,6 +1935,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
+ MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ }
+
// NOTE: Currently this has to happen after computeRegisterProperties rather
// than the preferred option of combining it with the addRegisterClass call.
if (Subtarget->useSVEForFixedLengthVectors()) {
@@ -1962,9 +1968,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// These operations are not supported on NEON but SVE can do them.
setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
- setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
@@ -1982,7 +1985,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::CTTZ, VT, Custom);
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
@@ -11705,7 +11707,7 @@ SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v2i32 ||
VT == MVT::v4i32 ||
useSVEForFixedLengthVectorVT(
- VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
+ VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()));
SDLoc DL(Op);
SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index a75a6a06e994f..2032434880995 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -16,7 +16,10 @@ define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: ctlz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.b, vl8
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -30,7 +33,10 @@ define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: ctlz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -44,7 +50,10 @@ define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: ctlz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: ptrue p0.s, vl2
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -74,21 +83,9 @@ define <1 x i64> @ctlz_v1i64(<1 x i64> %a) {
;
; SVE-LABEL: ctlz_v1i64:
; SVE: // %bb.0:
-; SVE-NEXT: ushr d1, d0, #1
; SVE-NEXT: ptrue p0.d, vl1
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #2
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #4
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #8
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #16
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: ushr d1, d0, #32
-; SVE-NEXT: orr v0.8b, v0.8b, v1.8b
-; SVE-NEXT: mvn v0.8b, v0.8b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false)
@@ -105,7 +102,10 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: ctlz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.b, vl16
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -119,7 +119,10 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: ctlz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -133,7 +136,10 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: ctlz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -163,21 +169,9 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
;
; SVE-LABEL: ctlz_v2i64:
; SVE: // %bb.0:
-; SVE-NEXT: ushr v1.2d, v0.2d, #1
; SVE-NEXT: ptrue p0.d, vl2
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #2
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #4
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #8
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #16
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: ushr v1.2d, v0.2d, #32
-; SVE-NEXT: orr v0.16b, v0.16b, v1.16b
-; SVE-NEXT: mvn v0.16b, v0.16b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false)
@@ -201,10 +195,10 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.8b, #1
-; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.b, vl8
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -223,12 +217,11 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.4h, #1
-; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.4h, #16
-; SVE-NEXT: clz v0.4h, v0.4h
-; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; SVE-NEXT: rev16 v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -247,12 +240,11 @@ define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.2s, #1
-; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.2s, #32
-; SVE-NEXT: clz v0.2s, v0.2s
-; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; SVE-NEXT: rev32 v0.8b, v0.8b
+; SVE-NEXT: ptrue p0.s, vl2
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -273,12 +265,10 @@ define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
;
; SVE-LABEL: cttz_v1i64:
; SVE: // %bb.0:
-; SVE-NEXT: mov w8, #1 // =0x1
+; SVE-NEXT: rev64 v0.8b, v0.8b
; SVE-NEXT: ptrue p0.d, vl1
-; SVE-NEXT: fmov d1, x8
-; SVE-NEXT: sub d1, d0, d1
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT: ret
%r = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 false)
@@ -298,11 +288,10 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.b, vl16
+; SVE-NEXT: clz z0.b, p0/m, z0.b
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -321,13 +310,11 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
-; SVE-NEXT: movi v1.8h, #16
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
-; SVE-NEXT: clz v0.8h, v0.8h
-; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
+; SVE-NEXT: rev16 v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.h, p0/m, z0.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -346,13 +333,11 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
-; SVE-NEXT: movi v1.4s, #32
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
-; SVE-NEXT: clz v0.4s, v0.4s
-; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
+; SVE-NEXT: rev32 v0.16b, v0.16b
+; SVE-NEXT: ptrue p0.s, vl4
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.s, p0/m, z0.s
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -373,12 +358,10 @@ define <2 x i64> @cttz_v2i64(<2 x i64> %a) {
;
; SVE-LABEL: cttz_v2i64:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.d, z1.d, #1 // =0x1
+; SVE-NEXT: rev64 v0.16b, v0.16b
; SVE-NEXT: ptrue p0.d, vl2
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt z0.d, p0/m, z0.d
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz z0.d, p0/m, z0.d
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
%r = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
>From 016dba1525c1f891873abb5fada66d1ca7b13f87 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 05:22:28 -0700
Subject: [PATCH 3/5] clang-format
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a45ff186a658c..9084af1baac5e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1935,10 +1935,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
- MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::CTTZ, VT, Custom);
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
}
// NOTE: Currently this has to happen after computeRegisterProperties rather
>From 964fa5af32776dc0e7e2d567de9d2a9e2bde31ee Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 21:30:09 -0700
Subject: [PATCH 4/5] Only lower vNi64 with sve
---
.../Target/AArch64/AArch64ISelLowering.cpp | 7 +-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 93 ++++++++-----------
2 files changed, 46 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9084af1baac5e..7c6871631e4d8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1935,8 +1935,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
- MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ for (auto VT : {MVT::v1i64, MVT::v2i64}) {
setOperationAction(ISD::CTLZ, VT, Custom);
setOperationAction(ISD::CTTZ, VT, Custom);
}
@@ -1989,6 +1988,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
}
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
+ MVT::v2i32, MVT::v4i32}) {
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ }
// Use SVE for vectors with more than 2 elements.
for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index 2032434880995..92da881f2b3bf 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -16,10 +16,7 @@ define <8 x i8> @ctlz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: ctlz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.b, vl8
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -33,10 +30,7 @@ define <4 x i16> @ctlz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: ctlz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.h, vl4
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -50,10 +44,7 @@ define <2 x i32> @ctlz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: ctlz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s, vl2
-; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: clz v0.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -102,10 +93,7 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: ctlz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.b, vl16
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -119,10 +107,7 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: ctlz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.h, vl8
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -136,10 +121,7 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: ctlz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: clz v0.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
@@ -195,10 +177,10 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.b, vl8
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.8b, #1
+; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: cnt v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -217,11 +199,12 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: rev16 v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.h, vl4
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.4h, #1
+; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: clz v0.4h, v0.4h
+; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -240,11 +223,12 @@ define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: rev32 v0.8b, v0.8b
-; SVE-NEXT: ptrue p0.s, vl2
-; SVE-NEXT: rbit v0.8b, v0.8b
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: movi v1.2s, #1
+; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
+; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
+; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: clz v0.2s, v0.2s
+; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -288,10 +272,11 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.b, vl16
-; SVE-NEXT: clz z0.b, p0/m, z0.b
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z1, z0
+; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
+; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
+; SVE-NEXT: cnt v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -310,11 +295,13 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: rev16 v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.h, vl8
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: clz z0.h, p0/m, z0.h
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
+; SVE-NEXT: movi v1.8h, #16
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.8h, v0.8h
+; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -333,11 +320,13 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: rev32 v0.16b, v0.16b
-; SVE-NEXT: ptrue p0.s, vl4
-; SVE-NEXT: rbit v0.16b, v0.16b
-; SVE-NEXT: clz z0.s, p0/m, z0.s
-; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: movprfx z2, z0
+; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
+; SVE-NEXT: movi v1.4s, #32
+; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: clz v0.4s, v0.4s
+; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
>From 5efa833d3b6f75d21e28c2585f339e948c71b5f2 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Thu, 16 Apr 2026 22:10:22 -0700
Subject: [PATCH 5/5] Custom lowering path for fixedlen_vector_valuetypes cttz
---
.../Target/AArch64/AArch64ISelLowering.cpp | 11 ++-
.../AArch64/ctlz-cttz-fixed-vectors.ll | 83 ++++++-------------
2 files changed, 29 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7c6871631e4d8..2adcdc5e9dffe 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1988,8 +1988,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
}
- for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
- MVT::v2i32, MVT::v4i32}) {
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32}) {
setOperationAction(ISD::CTTZ, VT, Custom);
}
@@ -11706,11 +11706,10 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
- assert(VT.isScalableVector() || VT == MVT::v8i8 || VT == MVT::v16i8 ||
- VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v2i32 ||
- VT == MVT::v4i32 ||
+ assert(VT.isScalableVector() ||
useSVEForFixedLengthVectorVT(
- VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()));
+ VT, /*OverrideNEON=*/Subtarget->isSVEorStreamingSVEAvailable()) ||
+ VT.getScalarSizeInBits() < 64);
SDLoc DL(Op);
SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
diff --git a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
index 92da881f2b3bf..ba313e4fb2fdc 100644
--- a/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz-cttz-fixed-vectors.ll
@@ -169,18 +169,14 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %a) {
define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
; NEON-LABEL: cttz_v8i8:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.8b, #1
-; NEON-NEXT: sub v1.8b, v0.8b, v1.8b
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: cnt v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
+; NEON-NEXT: clz v0.8b, v0.8b
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v8i8:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.8b, #1
-; SVE-NEXT: sub v1.8b, v0.8b, v1.8b
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: cnt v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
+; SVE-NEXT: clz v0.8b, v0.8b
; SVE-NEXT: ret
%r = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 false)
ret <8 x i8> %r
@@ -189,22 +185,16 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %a) {
define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
; NEON-LABEL: cttz_v4i16:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.4h, #1
-; NEON-NEXT: sub v1.4h, v0.4h, v1.4h
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: movi v1.4h, #16
+; NEON-NEXT: rev16 v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
; NEON-NEXT: clz v0.4h, v0.4h
-; NEON-NEXT: sub v0.4h, v1.4h, v0.4h
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v4i16:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.4h, #1
-; SVE-NEXT: sub v1.4h, v0.4h, v1.4h
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.4h, #16
+; SVE-NEXT: rev16 v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
; SVE-NEXT: clz v0.4h, v0.4h
-; SVE-NEXT: sub v0.4h, v1.4h, v0.4h
; SVE-NEXT: ret
%r = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 false)
ret <4 x i16> %r
@@ -213,22 +203,16 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %a) {
define <2 x i32> @cttz_v2i32(<2 x i32> %a) {
; NEON-LABEL: cttz_v2i32:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.2s, #1
-; NEON-NEXT: sub v1.2s, v0.2s, v1.2s
-; NEON-NEXT: bic v0.8b, v1.8b, v0.8b
-; NEON-NEXT: movi v1.2s, #32
+; NEON-NEXT: rev32 v0.8b, v0.8b
+; NEON-NEXT: rbit v0.8b, v0.8b
; NEON-NEXT: clz v0.2s, v0.2s
-; NEON-NEXT: sub v0.2s, v1.2s, v0.2s
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v2i32:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.2s, #1
-; SVE-NEXT: sub v1.2s, v0.2s, v1.2s
-; SVE-NEXT: bic v0.8b, v1.8b, v0.8b
-; SVE-NEXT: movi v1.2s, #32
+; SVE-NEXT: rev32 v0.8b, v0.8b
+; SVE-NEXT: rbit v0.8b, v0.8b
; SVE-NEXT: clz v0.2s, v0.2s
-; SVE-NEXT: sub v0.2s, v1.2s, v0.2s
; SVE-NEXT: ret
%r = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -264,19 +248,14 @@ define <1 x i64> @cttz_v1i64(<1 x i64> %a) {
define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
; NEON-LABEL: cttz_v16i8:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.16b, #1
-; NEON-NEXT: sub v1.16b, v0.16b, v1.16b
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: cnt v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
+; NEON-NEXT: clz v0.16b, v0.16b
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v16i8:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z1, z0
-; SVE-NEXT: sub z1.b, z1.b, #1 // =0x1
-; SVE-NEXT: bic v0.16b, v1.16b, v0.16b
-; SVE-NEXT: cnt v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
+; SVE-NEXT: clz v0.16b, v0.16b
; SVE-NEXT: ret
%r = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -285,23 +264,16 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %a) {
define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
; NEON-LABEL: cttz_v8i16:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.8h, #1
-; NEON-NEXT: sub v1.8h, v0.8h, v1.8h
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: movi v1.8h, #16
+; NEON-NEXT: rev16 v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
; NEON-NEXT: clz v0.8h, v0.8h
-; NEON-NEXT: sub v0.8h, v1.8h, v0.8h
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v8i16:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.h, z2.h, #1 // =0x1
-; SVE-NEXT: movi v1.8h, #16
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: rev16 v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
; SVE-NEXT: clz v0.8h, v0.8h
-; SVE-NEXT: sub v0.8h, v1.8h, v0.8h
; SVE-NEXT: ret
%r = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
ret <8 x i16> %r
@@ -310,23 +282,16 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %a) {
define <4 x i32> @cttz_v4i32(<4 x i32> %a) {
; NEON-LABEL: cttz_v4i32:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.4s, #1
-; NEON-NEXT: sub v1.4s, v0.4s, v1.4s
-; NEON-NEXT: bic v0.16b, v1.16b, v0.16b
-; NEON-NEXT: movi v1.4s, #32
+; NEON-NEXT: rev32 v0.16b, v0.16b
+; NEON-NEXT: rbit v0.16b, v0.16b
; NEON-NEXT: clz v0.4s, v0.4s
-; NEON-NEXT: sub v0.4s, v1.4s, v0.4s
; NEON-NEXT: ret
;
; SVE-LABEL: cttz_v4i32:
; SVE: // %bb.0:
-; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
-; SVE-NEXT: movprfx z2, z0
-; SVE-NEXT: sub z2.s, z2.s, #1 // =0x1
-; SVE-NEXT: movi v1.4s, #32
-; SVE-NEXT: bic v0.16b, v2.16b, v0.16b
+; SVE-NEXT: rev32 v0.16b, v0.16b
+; SVE-NEXT: rbit v0.16b, v0.16b
; SVE-NEXT: clz v0.4s, v0.4s
-; SVE-NEXT: sub v0.4s, v1.4s, v0.4s
; SVE-NEXT: ret
%r = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
ret <4 x i32> %r
More information about the llvm-commits
mailing list