[llvm] [AArch64][ISel] Add lowering for fixed-width `cttz` intrinsic (PR #190988)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 08:14:55 PDT 2026
https://github.com/Harry-Ramsey created https://github.com/llvm/llvm-project/pull/190988
This patch enables NEON to generate more efficient `cttz` intrinsics by utilising `rbit` and `ctlz` instructions when they are legal.
>From 70d06b7317b64d26c4d4b30c188214b3117734ba Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 2 Apr 2026 10:05:00 +0000
Subject: [PATCH] [AArch64][ISel] Add lowering for fixed-width `cttz` intrinsic
This patch enables NEON to generate more efficient `cttz` intrinsics by
utilising `rbit` and `ctlz` instructions when they are legal.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 8 +++++--
llvm/test/Analysis/CostModel/AArch64/cttz.ll | 6 ++---
llvm/test/CodeGen/AArch64/cttz.ll | 23 +++++++------------
3 files changed, 17 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 178842f9761cd..cf859498ad7b9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1422,7 +1422,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BSWAP, VT, Legal);
else
setOperationAction(ISD::BSWAP, VT, Expand);
- setOperationAction(ISD::CTTZ, VT, Expand);
+
+ if (VT == MVT::v8i8 || VT == MVT::v16i8)
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ else
+ setOperationAction(ISD::CTTZ, VT, Expand);
for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
setTruncStoreAction(VT, InnerVT, Expand);
@@ -12068,7 +12072,7 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
- assert(VT.isScalableVector() ||
+ assert(VT.isScalableVector() || VT == MVT::v8i8 || VT == MVT::v16i8 ||
useSVEForFixedLengthVectorVT(
VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz.ll b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
index 6f7810372388a..2c110eaeba269 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
@@ -135,7 +135,7 @@ define <4 x i8> @test_cttz_v4i8(<4 x i8> %a) {
define <8 x i8> @test_cttz_v8i8(<8 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %cttz
;
%cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
@@ -145,7 +145,7 @@ define <8 x i8> @test_cttz_v8i8(<8 x i8> %a) {
define <16 x i8> @test_cttz_v16i8(<16 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v16i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:80 SizeLat:80 for: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %cttz
;
%cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
@@ -185,7 +185,7 @@ define <16 x i16> @test_cttz_v16i16(<16 x i16> %a) {
define <32 x i8> @test_cttz_v32i8(<32 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v32i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:96 Lat:160 SizeLat:160 for: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <32 x i8> %cttz
;
%cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index c9181b4c312d1..55bad9c98bead 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -149,10 +149,8 @@ entry:
define <8 x i8> @v8i8(<8 x i8> %d) {
; CHECK-SD-LABEL: v8i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.8b, #1
-; CHECK-SD-NEXT: sub v1.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: cnt v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
+; CHECK-SD-NEXT: clz v0.8b, v0.8b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i8:
@@ -170,10 +168,8 @@ entry:
define <16 x i8> @v16i8(<16 x i8> %d) {
; CHECK-SD-LABEL: v16i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.16b, #1
-; CHECK-SD-NEXT: sub v1.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v16i8:
@@ -191,13 +187,10 @@ entry:
define <32 x i8> @v32i8(<32 x i8> %d) {
; CHECK-SD-LABEL: v32i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.16b, #1
-; CHECK-SD-NEXT: sub v3.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: sub v2.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
-; CHECK-SD-NEXT: cnt v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v1.16b, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v32i8:
More information about the llvm-commits
mailing list