[llvm] [AArch64][ISel] Add lowering for fixed-width `cttz` intrinsic (PR #190988)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 03:53:03 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/190988
>From 70d06b7317b64d26c4d4b30c188214b3117734ba Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 2 Apr 2026 10:05:00 +0000
Subject: [PATCH 1/4] [AArch64][ISel] Add lowering for fixed-width `cttz`
intrinsic
This patch enables NEON to generate more efficient `cttz` intrinsics by
utilising `rbit` and `ctlz` instructions when they are legal.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 8 +++++--
llvm/test/Analysis/CostModel/AArch64/cttz.ll | 6 ++---
llvm/test/CodeGen/AArch64/cttz.ll | 23 +++++++------------
3 files changed, 17 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 178842f9761cd..cf859498ad7b9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1422,7 +1422,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BSWAP, VT, Legal);
else
setOperationAction(ISD::BSWAP, VT, Expand);
- setOperationAction(ISD::CTTZ, VT, Expand);
+
+ if (VT == MVT::v8i8 || VT == MVT::v16i8)
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ else
+ setOperationAction(ISD::CTTZ, VT, Expand);
for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
setTruncStoreAction(VT, InnerVT, Expand);
@@ -12068,7 +12072,7 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
- assert(VT.isScalableVector() ||
+ assert(VT.isScalableVector() || VT == MVT::v8i8 || VT == MVT::v16i8 ||
useSVEForFixedLengthVectorVT(
VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz.ll b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
index 6f7810372388a..2c110eaeba269 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
@@ -135,7 +135,7 @@ define <4 x i8> @test_cttz_v4i8(<4 x i8> %a) {
define <8 x i8> @test_cttz_v8i8(<8 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %cttz
;
%cttz = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
@@ -145,7 +145,7 @@ define <8 x i8> @test_cttz_v8i8(<8 x i8> %a) {
define <16 x i8> @test_cttz_v16i8(<16 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v16i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:80 SizeLat:80 for: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %cttz
;
%cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
@@ -185,7 +185,7 @@ define <16 x i16> @test_cttz_v16i16(<16 x i16> %a) {
define <32 x i8> @test_cttz_v32i8(<32 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v32i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:96 Lat:160 SizeLat:160 for: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <32 x i8> %cttz
;
%cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index c9181b4c312d1..55bad9c98bead 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -149,10 +149,8 @@ entry:
define <8 x i8> @v8i8(<8 x i8> %d) {
; CHECK-SD-LABEL: v8i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.8b, #1
-; CHECK-SD-NEXT: sub v1.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: cnt v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
+; CHECK-SD-NEXT: clz v0.8b, v0.8b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i8:
@@ -170,10 +168,8 @@ entry:
define <16 x i8> @v16i8(<16 x i8> %d) {
; CHECK-SD-LABEL: v16i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.16b, #1
-; CHECK-SD-NEXT: sub v1.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v16i8:
@@ -191,13 +187,10 @@ entry:
define <32 x i8> @v32i8(<32 x i8> %d) {
; CHECK-SD-LABEL: v32i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.16b, #1
-; CHECK-SD-NEXT: sub v3.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: sub v2.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: cnt v0.16b, v0.16b
-; CHECK-SD-NEXT: cnt v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
+; CHECK-SD-NEXT: clz v0.16b, v0.16b
+; CHECK-SD-NEXT: clz v1.16b, v1.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v32i8:
>From 654d1840c7025234c8a93d53d6204ef813618285 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 9 Apr 2026 10:49:38 +0000
Subject: [PATCH 2/4] fixup! [AArch64][ISel] Add lowering for fixed-width
`cttz` intrinsic
---
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
llvm/test/Analysis/CostModel/AArch64/cttz.ll | 18 ++--
llvm/test/CodeGen/AArch64/cttz.ll | 96 ++++++-------------
3 files changed, 42 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cf859498ad7b9..e6b79becf1d2f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1423,7 +1423,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
else
setOperationAction(ISD::BSWAP, VT, Expand);
- if (VT == MVT::v8i8 || VT == MVT::v16i8)
+ if (VT == MVT::v8i8 || VT == MVT::v16i8 || VT == MVT::v8i16 ||
+ VT == MVT::v4i16 || VT == MVT::v2i32 || VT == MVT::v4i32)
setOperationAction(ISD::CTTZ, VT, Custom);
else
setOperationAction(ISD::CTTZ, VT, Expand);
@@ -12073,6 +12074,8 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
assert(VT.isScalableVector() || VT == MVT::v8i8 || VT == MVT::v16i8 ||
+ VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v2i32 ||
+ VT == MVT::v4i32 ||
useSVEForFixedLengthVectorVT(
VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz.ll b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
index 2c110eaeba269..f4696d2442e0f 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
@@ -65,7 +65,7 @@ define <2 x i64> @test_cttz_v2i64(<2 x i64> %a) {
define <2 x i32> @test_cttz_v2i32(<2 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:10 SizeLat:10 for: %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i32> %cttz
;
%cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
@@ -75,7 +75,7 @@ define <2 x i32> @test_cttz_v2i32(<2 x i32> %a) {
define <4 x i32> @test_cttz_v4i32(<4 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %cttz
;
%cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
@@ -85,7 +85,7 @@ define <4 x i32> @test_cttz_v4i32(<4 x i32> %a) {
define <2 x i16> @test_cttz_v2i16(<2 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:10 SizeLat:10 for: %cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i16> %cttz
;
%cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
@@ -95,7 +95,7 @@ define <2 x i16> @test_cttz_v2i16(<2 x i16> %a) {
define <4 x i16> @test_cttz_v4i16(<4 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %cttz
;
%cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
@@ -105,7 +105,7 @@ define <4 x i16> @test_cttz_v4i16(<4 x i16> %a) {
define <8 x i16> @test_cttz_v8i16(<8 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %cttz
;
%cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
@@ -115,7 +115,7 @@ define <8 x i16> @test_cttz_v8i16(<8 x i16> %a) {
define <2 x i8> @test_cttz_v2i8(<2 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:10 SizeLat:10 for: %cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i8> %cttz
;
%cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
@@ -125,7 +125,7 @@ define <2 x i8> @test_cttz_v2i8(<2 x i8> %a) {
define <4 x i8> @test_cttz_v4i8(<4 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i8'
-; CHECK-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %cttz
;
%cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
@@ -165,7 +165,7 @@ define <4 x i64> @test_cttz_v4i64(<4 x i64> %a) {
define <8 x i32> @test_cttz_v8i32(<8 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i32'
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i32> %cttz
;
%cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
@@ -175,7 +175,7 @@ define <8 x i32> @test_cttz_v8i32(<8 x i32> %a) {
define <16 x i16> @test_cttz_v16i16(<16 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v16i16'
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:48 Lat:80 SizeLat:80 for: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %cttz
;
%cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
index 55bad9c98bead..86c1aca9ed9e1 100644
--- a/llvm/test/CodeGen/AArch64/cttz.ll
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -7,15 +7,12 @@ define void @v2i8(ptr %p1) {
; CHECK-SD-LABEL: v2i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr h0, [x0]
-; CHECK-SD-NEXT: movi v1.2s, #1
; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: orr v0.2s, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: mov s1, v0.s[1]
; CHECK-SD-NEXT: str b0, [x0]
; CHECK-SD-NEXT: stur b1, [x0, #1]
@@ -49,14 +46,11 @@ define void @v3i8(ptr %p1) {
; CHECK-SD-NEXT: sub sp, sp, #16
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.4h, #1
; CHECK-SD-NEXT: zip1 v0.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: orr v0.4h, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: uzp1 v1.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: mov h0, v0.h[2]
; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0
@@ -101,14 +95,11 @@ define void @v4i8(ptr %p1) {
; CHECK-SD-LABEL: v4i8:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.4h, #1
; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: orr v0.4h, #1, lsl #8
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b
; CHECK-SD-NEXT: str s0, [x0]
; CHECK-SD-NEXT: ret
@@ -212,14 +203,11 @@ define void @v2i16(ptr %p1) {
; CHECK-SD-LABEL: v2i16:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ldr s0, [x0]
-; CHECK-SD-NEXT: movi v1.2s, #1
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: orr v0.2s, #1, lsl #16
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: mov s1, v0.s[1]
; CHECK-SD-NEXT: str h0, [x0]
; CHECK-SD-NEXT: str h1, [x0, #2]
@@ -253,13 +241,10 @@ entry:
define void @v3i16(ptr %p1) {
; CHECK-SD-LABEL: v3i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v0.4h, #1
-; CHECK-SD-NEXT: ldr d1, [x0]
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
-; CHECK-SD-NEXT: bic v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: ldr d0, [x0]
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: mov h1, v0.h[2]
; CHECK-SD-NEXT: str s0, [x0]
; CHECK-SD-NEXT: str h1, [x0, #4]
@@ -293,12 +278,9 @@ entry:
define <4 x i16> @v4i16(<4 x i16> %d) {
; CHECK-SD-LABEL: v4i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4h, #1
-; CHECK-SD-NEXT: sub v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.4h, #16
+; CHECK-SD-NEXT: rev16 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.4h, v0.4h
-; CHECK-SD-NEXT: sub v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v4i16:
@@ -317,12 +299,9 @@ entry:
define <8 x i16> @v8i16(<8 x i16> %d) {
; CHECK-SD-LABEL: v8i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.8h, #1
-; CHECK-SD-NEXT: sub v1.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.8h, #16
+; CHECK-SD-NEXT: rev16 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.8h, v0.8h
-; CHECK-SD-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i16:
@@ -341,16 +320,12 @@ entry:
define <16 x i16> @v16i16(<16 x i16> %d) {
; CHECK-SD-LABEL: v16i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.8h, #1
-; CHECK-SD-NEXT: sub v3.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT: sub v2.8h, v1.8h, v2.8h
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: movi v2.8h, #16
+; CHECK-SD-NEXT: rev16 v0.16b, v0.16b
+; CHECK-SD-NEXT: rev16 v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
; CHECK-SD-NEXT: clz v0.8h, v0.8h
; CHECK-SD-NEXT: clz v1.8h, v1.8h
-; CHECK-SD-NEXT: sub v0.8h, v2.8h, v0.8h
-; CHECK-SD-NEXT: sub v1.8h, v2.8h, v1.8h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v16i16:
@@ -373,12 +348,9 @@ entry:
define <2 x i32> @v2i32(<2 x i32> %d) {
; CHECK-SD-LABEL: v2i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.2s, #1
-; CHECK-SD-NEXT: sub v1.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: bic v0.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT: movi v1.2s, #32
+; CHECK-SD-NEXT: rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT: rbit v0.8b, v0.8b
; CHECK-SD-NEXT: clz v0.2s, v0.2s
-; CHECK-SD-NEXT: sub v0.2s, v1.2s, v0.2s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v2i32:
@@ -398,12 +370,9 @@ entry:
define <3 x i32> @v3i32(<3 x i32> %d) {
; CHECK-SD-LABEL: v3i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4s, #1
-; CHECK-SD-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
-; CHECK-SD-NEXT: sub v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v3i32:
@@ -427,12 +396,9 @@ entry:
define <4 x i32> @v4i32(<4 x i32> %d) {
; CHECK-SD-LABEL: v4i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v1.4s, #1
-; CHECK-SD-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b
-; CHECK-SD-NEXT: movi v1.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
-; CHECK-SD-NEXT: sub v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v4i32:
@@ -452,16 +418,12 @@ entry:
define <8 x i32> @v8i32(<8 x i32> %d) {
; CHECK-SD-LABEL: v8i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: movi v2.4s, #1
-; CHECK-SD-NEXT: sub v3.4s, v0.4s, v2.4s
-; CHECK-SD-NEXT: sub v2.4s, v1.4s, v2.4s
-; CHECK-SD-NEXT: bic v0.16b, v3.16b, v0.16b
-; CHECK-SD-NEXT: bic v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT: movi v2.4s, #32
+; CHECK-SD-NEXT: rev32 v0.16b, v0.16b
+; CHECK-SD-NEXT: rev32 v1.16b, v1.16b
+; CHECK-SD-NEXT: rbit v0.16b, v0.16b
+; CHECK-SD-NEXT: rbit v1.16b, v1.16b
; CHECK-SD-NEXT: clz v0.4s, v0.4s
; CHECK-SD-NEXT: clz v1.4s, v1.4s
-; CHECK-SD-NEXT: sub v0.4s, v2.4s, v0.4s
-; CHECK-SD-NEXT: sub v1.4s, v2.4s, v1.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: v8i32:
>From 4c0ea2a5db415556b8342185befe3efcdfb52310 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Mon, 13 Apr 2026 08:13:29 +0000
Subject: [PATCH 3/4] amend! [AArch64][ISel] Add lowering for fixed-width
`cttz` intrinsic
[AArch64][ISel] Add lowering for fixed-width `cttz` intrinsic
This patch enables NEON to generate more efficient `cttz` intrinsics by
utilising `rbit` and `ctlz` instructions when they are legal.
Additionally improve cost analysis for cttz by having it be more
accurate for the number of instructions produced.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 13 +++++++++++++
llvm/test/Analysis/CostModel/AArch64/cttz.ll | 18 +++++++++---------
2 files changed, 22 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 734339e5c7a05..4326d60344ae5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -29,6 +29,9 @@
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <optional>
+
+#include "llvm/Support/raw_ostream.h"
+
using namespace llvm;
using namespace llvm::PatternMatch;
@@ -1076,6 +1079,16 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
}
break;
}
+ case Intrinsic::cttz: {
+ auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
+ // llvm::errs() << "LT Value: " << LT.first << "\n";
+ if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
+ return LT.first * 2;
+ if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
+ LT.second == MVT::v2i32 || LT.second == MVT::v4i32)
+ return LT.first * 3;
+ break;
+ }
case Intrinsic::experimental_cttz_elts: {
EVT ArgVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]);
if (!getTLI()->shouldExpandCttzElements(ArgVT)) {
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz.ll b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
index f4696d2442e0f..b890b31f5c5b1 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
@@ -65,7 +65,7 @@ define <2 x i64> @test_cttz_v2i64(<2 x i64> %a) {
define <2 x i32> @test_cttz_v2i32(<2 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i32'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i32> %cttz
;
%cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
@@ -75,7 +75,7 @@ define <2 x i32> @test_cttz_v2i32(<2 x i32> %a) {
define <4 x i32> @test_cttz_v4i32(<4 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i32'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %cttz
;
%cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
@@ -85,7 +85,7 @@ define <4 x i32> @test_cttz_v4i32(<4 x i32> %a) {
define <2 x i16> @test_cttz_v2i16(<2 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i16'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i16> %cttz
;
%cttz = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %a, i1 true)
@@ -95,7 +95,7 @@ define <2 x i16> @test_cttz_v2i16(<2 x i16> %a) {
define <4 x i16> @test_cttz_v4i16(<4 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i16'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %cttz
;
%cttz = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
@@ -105,7 +105,7 @@ define <4 x i16> @test_cttz_v4i16(<4 x i16> %a) {
define <8 x i16> @test_cttz_v8i16(<8 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i16'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %cttz
;
%cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
@@ -115,7 +115,7 @@ define <8 x i16> @test_cttz_v8i16(<8 x i16> %a) {
define <2 x i8> @test_cttz_v2i8(<2 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v2i8'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i8> %cttz
;
%cttz = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %a, i1 true)
@@ -125,7 +125,7 @@ define <2 x i8> @test_cttz_v2i8(<2 x i8> %a) {
define <4 x i8> @test_cttz_v4i8(<4 x i8> %a) {
;
; CHECK-LABEL: 'test_cttz_v4i8'
-; CHECK-NEXT: Cost Model: Found costs of 2 for: %cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 3 for: %cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %cttz
;
%cttz = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %a, i1 true)
@@ -165,7 +165,7 @@ define <4 x i64> @test_cttz_v4i64(<4 x i64> %a) {
define <8 x i32> @test_cttz_v8i32(<8 x i32> %a) {
;
; CHECK-LABEL: 'test_cttz_v8i32'
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i32> %cttz
;
%cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
@@ -175,7 +175,7 @@ define <8 x i32> @test_cttz_v8i32(<8 x i32> %a) {
define <16 x i16> @test_cttz_v16i16(<16 x i16> %a) {
;
; CHECK-LABEL: 'test_cttz_v16i16'
-; CHECK-NEXT: Cost Model: Found costs of 4 for: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; CHECK-NEXT: Cost Model: Found costs of 6 for: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %cttz
;
%cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
>From 0c399cb146b0c633b49e7784b59a64fd03808708 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 14 Apr 2026 10:31:21 +0000
Subject: [PATCH 4/4] fixup! [AArch64][ISel] Add lowering for fixed-width
`cttz` intrinsic
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4326d60344ae5..0588fbc68b1d4 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -29,9 +29,6 @@
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <optional>
-
-#include "llvm/Support/raw_ostream.h"
-
using namespace llvm;
using namespace llvm::PatternMatch;
@@ -1081,7 +1078,6 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
}
case Intrinsic::cttz: {
auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
- // llvm::errs() << "LT Value: " << LT.first << "\n";
if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
return LT.first * 2;
if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
More information about the llvm-commits
mailing list