[llvm] [AArch64][ISel] Prefer SVE over NEON for CTLZ (PR #194839)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Thu May 7 08:03:50 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/194839
>From 382d957f5390ddd764a5a8db4f0fb6d47a41d652 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 28 Apr 2026 11:45:10 +0000
Subject: [PATCH 1/2] [AArch64][ISel] Prefer SVE over NEON for CTLZ
This commit enables SVE instructions for i64 data types that are
unsupported on NEON but supported on SVE.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8c53ea0dd94eb..dde73d280a15c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1931,9 +1931,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::MULHU, VT, Custom);
}
- // NEON doesn't support 64-bit vector integer muls, but SVE does.
+ // NEON doesn't support 64-bit vector integer mul and ctlz operations, but
+ // SVE does.
setOperationAction(ISD::MUL, MVT::v1i64, Custom);
setOperationAction(ISD::MUL, MVT::v2i64, Custom);
+ setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
+ setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
// With SVE2 we can try lowering these to pairwise operations (e.g. smaxp).
if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
@@ -1970,8 +1973,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// These operations are not supported on NEON but SVE can do them.
setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
- setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
>From b0dde3e234075ef12bc57ddd01a42edff276557b Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 30 Apr 2026 08:24:50 +0000
Subject: [PATCH 2/2] amend! [AArch64][ISel] Use SVE CTLZ when no NEON
equivalent exists
[AArch64][ISel] Use SVE CTLZ when no NEON equivalent exists
This commit enables SVE instructions for i64 data types that do not have
equivalent instructions on NEON but supported on SVE.
---
llvm/test/CodeGen/AArch64/ctlz.ll | 166 +++++++++++++++++++++++++++++-
1 file changed, 164 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
index 5740eaa977875..b764a9a1489d0 100644
--- a/llvm/test/CodeGen/AArch64/ctlz.ll
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
define void @v2i8(ptr %p1) {
; CHECK-SD-LABEL: v2i8:
; CHECK-SD: // %bb.0: // %entry
@@ -26,6 +25,19 @@ define void @v2i8(ptr %p1) {
; CHECK-GI-NEXT: st1 { v0.b }[0], [x0]
; CHECK-GI-NEXT: st1 { v0.b }[1], [x8]
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v2i8:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ldr h1, [x0]
+; CHECK-SVE-NEXT: movi v0.2s, #24
+; CHECK-SVE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SVE-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SVE-NEXT: clz v1.2s, v1.2s
+; CHECK-SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; CHECK-SVE-NEXT: mov s1, v0.s[1]
+; CHECK-SVE-NEXT: str b0, [x0]
+; CHECK-SVE-NEXT: stur b1, [x0, #1]
+; CHECK-SVE-NEXT: ret
entry:
%d = load <2 x i8>, ptr %p1
%s = call <2 x i8> @llvm.ctlz(<2 x i8> %d, i1 false)
@@ -68,6 +80,24 @@ define void @v3i8(ptr %p1) {
; CHECK-GI-NEXT: st1 { v0.b }[1], [x8]
; CHECK-GI-NEXT: st1 { v0.b }[2], [x9]
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v3i8:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: sub sp, sp, #16
+; CHECK-SVE-NEXT: .cfi_def_cfa_offset 16
+; CHECK-SVE-NEXT: movi v0.4h, #8
+; CHECK-SVE-NEXT: ldr s1, [x0]
+; CHECK-SVE-NEXT: zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SVE-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-SVE-NEXT: clz v1.4h, v1.4h
+; CHECK-SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; CHECK-SVE-NEXT: uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SVE-NEXT: mov h0, v0.h[2]
+; CHECK-SVE-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SVE-NEXT: stur b0, [x0, #2]
+; CHECK-SVE-NEXT: str h1, [x0]
+; CHECK-SVE-NEXT: add sp, sp, #16
+; CHECK-SVE-NEXT: ret
entry:
%d = load <3 x i8>, ptr %p1
%s = call <3 x i8> @llvm.ctlz(<3 x i8> %d, i1 false)
@@ -101,6 +131,17 @@ define void @v4i8(ptr %p1) {
; CHECK-GI-NEXT: clz v0.8b, v2.8b
; CHECK-GI-NEXT: str s0, [x0]
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v4i8:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ldr s1, [x0]
+; CHECK-SVE-NEXT: movi v0.4h, #8
+; CHECK-SVE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SVE-NEXT: clz v1.4h, v1.4h
+; CHECK-SVE-NEXT: sub v0.4h, v1.4h, v0.4h
+; CHECK-SVE-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-SVE-NEXT: str s0, [x0]
+; CHECK-SVE-NEXT: ret
entry:
%d = load <4 x i8>, ptr %p1
%s = call <4 x i8> @llvm.ctlz(<4 x i8> %d, i1 false)
@@ -161,6 +202,18 @@ define void @v2i16(ptr %p1) {
; CHECK-GI-NEXT: str h0, [x0]
; CHECK-GI-NEXT: st1 { v0.h }[1], [x8]
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v2i16:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ldr s1, [x0]
+; CHECK-SVE-NEXT: movi v0.2s, #16
+; CHECK-SVE-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SVE-NEXT: clz v1.2s, v1.2s
+; CHECK-SVE-NEXT: sub v0.2s, v1.2s, v0.2s
+; CHECK-SVE-NEXT: mov s1, v0.s[1]
+; CHECK-SVE-NEXT: str h0, [x0]
+; CHECK-SVE-NEXT: str h1, [x0, #2]
+; CHECK-SVE-NEXT: ret
entry:
%d = load <2 x i16>, ptr %p1
%s = call <2 x i16> @llvm.ctlz(<2 x i16> %d, i1 false)
@@ -188,6 +241,15 @@ define void @v3i16(ptr %p1) {
; CHECK-GI-NEXT: st1 { v0.h }[1], [x8]
; CHECK-GI-NEXT: st1 { v0.h }[2], [x9]
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v3i16:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ldr d0, [x0]
+; CHECK-SVE-NEXT: clz v0.4h, v0.4h
+; CHECK-SVE-NEXT: mov h1, v0.h[2]
+; CHECK-SVE-NEXT: str s0, [x0]
+; CHECK-SVE-NEXT: str h1, [x0, #4]
+; CHECK-SVE-NEXT: ret
entry:
%d = load <3 x i16>, ptr %p1
%s = call <3 x i16> @llvm.ctlz(<3 x i16> %d, i1 false)
@@ -298,6 +360,14 @@ define <2 x i64> @v2i64(<2 x i64> %d) {
; CHECK-GI-NEXT: clz x8, x8
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v2i64:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE-NEXT: clz z0.d, p0/m, z0.d
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
entry:
%s = call <2 x i64> @llvm.ctlz(<2 x i64> %d, i1 false)
ret <2 x i64> %s
@@ -362,6 +432,21 @@ define <3 x i64> @v3i64(<3 x i64> %d) {
; CHECK-GI-NEXT: fmov d1, x9
; CHECK-GI-NEXT: fmov d2, x10
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v3i64:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-SVE-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: // kill: def $d2 killed $d2 def $z2
+; CHECK-SVE-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SVE-NEXT: clz z2.d, p0/m, z2.d
+; CHECK-SVE-NEXT: // kill: def $d2 killed $d2 killed $z2
+; CHECK-SVE-NEXT: clz z0.d, p0/m, z0.d
+; CHECK-SVE-NEXT: ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-SVE-NEXT: ret
entry:
%s = call <3 x i64> @llvm.ctlz(<3 x i64> %d, i1 false)
ret <3 x i64> %s
@@ -421,6 +506,17 @@ define <4 x i64> @v4i64(<4 x i64> %d) {
; CHECK-GI-NEXT: mov v0.d[1], x8
; CHECK-GI-NEXT: mov v1.d[1], x9
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v4i64:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE-NEXT: clz z0.d, p0/m, z0.d
+; CHECK-SVE-NEXT: clz z1.d, p0/m, z1.d
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
+; CHECK-SVE-NEXT: ret
entry:
%s = call <4 x i64> @llvm.ctlz(<4 x i64> %d, i1 false)
ret <4 x i64> %s
@@ -463,6 +559,22 @@ define <2 x i128> @v2i128(<2 x i128> %d) {
; CHECK-GI-NEXT: csel x2, x9, x10, eq
; CHECK-GI-NEXT: csel x3, x8, xzr, eq
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v2i128:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: clz x8, x0
+; CHECK-SVE-NEXT: clz x9, x1
+; CHECK-SVE-NEXT: cmp x1, #0
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: clz x10, x2
+; CHECK-SVE-NEXT: mov x1, xzr
+; CHECK-SVE-NEXT: csel x0, x9, x8, ne
+; CHECK-SVE-NEXT: clz x8, x3
+; CHECK-SVE-NEXT: add x9, x10, #64
+; CHECK-SVE-NEXT: cmp x3, #0
+; CHECK-SVE-NEXT: mov x3, xzr
+; CHECK-SVE-NEXT: csel x2, x8, x9, ne
+; CHECK-SVE-NEXT: ret
entry:
%s = call <2 x i128> @llvm.ctlz(<2 x i128> %d, i1 false)
ret <2 x i128> %s
@@ -519,6 +631,28 @@ define <3 x i128> @v3i128(<3 x i128> %d) {
; CHECK-GI-NEXT: csel x4, x9, x10, eq
; CHECK-GI-NEXT: csel x5, x8, xzr, eq
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v3i128:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: clz x8, x0
+; CHECK-SVE-NEXT: clz x9, x1
+; CHECK-SVE-NEXT: cmp x1, #0
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: clz x10, x2
+; CHECK-SVE-NEXT: mov x1, xzr
+; CHECK-SVE-NEXT: csel x0, x9, x8, ne
+; CHECK-SVE-NEXT: clz x8, x3
+; CHECK-SVE-NEXT: add x9, x10, #64
+; CHECK-SVE-NEXT: cmp x3, #0
+; CHECK-SVE-NEXT: mov x3, xzr
+; CHECK-SVE-NEXT: csel x2, x8, x9, ne
+; CHECK-SVE-NEXT: clz x8, x4
+; CHECK-SVE-NEXT: clz x9, x5
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: cmp x5, #0
+; CHECK-SVE-NEXT: mov x5, xzr
+; CHECK-SVE-NEXT: csel x4, x9, x8, ne
+; CHECK-SVE-NEXT: ret
entry:
%s = call <3 x i128> @llvm.ctlz(<3 x i128> %d, i1 false)
ret <3 x i128> %s
@@ -589,6 +723,34 @@ define <4 x i128> @v4i128(<4 x i128> %d) {
; CHECK-GI-NEXT: csel x6, x9, x10, eq
; CHECK-GI-NEXT: csel x7, x8, xzr, eq
; CHECK-GI-NEXT: ret
+;
+; CHECK-SVE-LABEL: v4i128:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: clz x9, x0
+; CHECK-SVE-NEXT: clz x8, x2
+; CHECK-SVE-NEXT: clz x10, x1
+; CHECK-SVE-NEXT: add x9, x9, #64
+; CHECK-SVE-NEXT: cmp x1, #0
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: csel x0, x10, x9, ne
+; CHECK-SVE-NEXT: clz x9, x3
+; CHECK-SVE-NEXT: cmp x3, #0
+; CHECK-SVE-NEXT: csel x2, x9, x8, ne
+; CHECK-SVE-NEXT: clz x8, x4
+; CHECK-SVE-NEXT: clz x9, x5
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: cmp x5, #0
+; CHECK-SVE-NEXT: mov x1, xzr
+; CHECK-SVE-NEXT: csel x4, x9, x8, ne
+; CHECK-SVE-NEXT: clz x8, x6
+; CHECK-SVE-NEXT: clz x9, x7
+; CHECK-SVE-NEXT: add x8, x8, #64
+; CHECK-SVE-NEXT: cmp x7, #0
+; CHECK-SVE-NEXT: mov x3, xzr
+; CHECK-SVE-NEXT: csel x6, x9, x8, ne
+; CHECK-SVE-NEXT: mov x5, xzr
+; CHECK-SVE-NEXT: mov x7, xzr
+; CHECK-SVE-NEXT: ret
entry:
%s = call <4 x i128> @llvm.ctlz(<4 x i128> %d, i1 false)
ret <4 x i128> %s
More information about the llvm-commits
mailing list