[llvm] [AArch64] Update cttz cost (PR #214003)

Tue Aug 4 09:51:27 PDT 2026

llvmorg-github-actions[bot] wrote:




@llvm/pr-subscribers-backend-aarch64

Author: David Green (davemgreen)

<details>
<summary>Changes</summary>

The previous scalar cttz cost was 1, even though we generate a rbit + clz (without cssc which adds a ctz instruction). Add some costs for scalar instructions whilst trying to keep the vector costs mostly the same.

---
Full diff: https://github.com/llvm/llvm-project/pull/214003.diff


2 Files Affected:

- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+25-6) 
- (modified) llvm/test/Analysis/CostModel/AArch64/cttz.ll (+21-16) 


``````````diff

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7b1c603d4134c..e08ed336ccfe2 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1163,12 +1163,31 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     break;
   }
   case Intrinsic::cttz: {
-    auto LT = getTypeLegalizationCost(ICA.getArgTypes()[0]);
-    if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
-      return LT.first * 2;
-    if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
-        LT.second == MVT::v2i32 || LT.second == MVT::v4i32)
-      return LT.first * 3;
+    auto LT = getTypeLegalizationCost(RetTy);
+    if (LT.second == MVT::i32 || LT.second == MVT::i64) {
+      // Extra cost for and mask of smaller types
+      InstructionCost ExtraCost =
+          LT.second.getSizeInBits() > RetTy->getScalarSizeInBits() ? 1 : 0;
+      // And combine larger sizes to i64 with cmp+add+csel
+      if (LT.second.getSizeInBits() < RetTy->getScalarSizeInBits())
+        ExtraCost += (LT.first - 1) * 3;
+      // Basic cost is rbit+clz or ctz.
+      return LT.first * (ST->hasCSSC() ? 1 : 2) + ExtraCost;
+    }
+
+    static const CostTblEntry BaseCostTbl[] = {
+        {Intrinsic::cttz, MVT::v8i8, 2}, // rbit+clz
+        {Intrinsic::cttz, MVT::v16i8, 2},
+        {Intrinsic::cttz, MVT::v4i16, 3}, // rev16+rbit+clz
+        {Intrinsic::cttz, MVT::v8i16, 3},
+        {Intrinsic::cttz, MVT::v2i32, 3},
+        {Intrinsic::cttz, MVT::v4i32, 3},
+        {Intrinsic::cttz, MVT::v1i64, 6}, // add+bic+cnt+reduce
+        {Intrinsic::cttz, MVT::v2i64, 6}};
+    const auto *Entry =
+        CostTableLookup(BaseCostTbl, Intrinsic::cttz, LT.second);
+    if (Entry)
+      return LT.first * Entry->Cost;
     break;
   }
   case Intrinsic::experimental_cttz_elts: {
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz.ll b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
index 68b69855cd247..a7e7233313d90 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz.ll
@@ -5,19 +5,30 @@
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @scalar() {
-; CHECK-LABEL: 'scalar'
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I8 = call i8 @llvm.cttz.i8(i8 undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I16 = call i16 @llvm.cttz.i16(i16 undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.cttz.i32(i32 undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I64 = call i64 @llvm.cttz.i64(i64 undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I128 = call i128 @llvm.cttz.i128(i128 undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-BASE-LABEL: 'scalar'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 3 for: %I8 = call i8 @llvm.cttz.i8(i8 undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 3 for: %I16 = call i16 @llvm.cttz.i16(i16 undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.cttz.i32(i32 undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.cttz.i64(i64 undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 7 for: %I128 = call i128 @llvm.cttz.i128(i128 undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 17 for: %V2I128 = call <2 x i128> @llvm.cttz.v2i128(<2 x i128> undef, i1 false)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-CSSC-LABEL: 'scalar'
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 2 for: %I8 = call i8 @llvm.cttz.i8(i8 undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 2 for: %I16 = call i16 @llvm.cttz.i16(i16 undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 1 for: %I32 = call i32 @llvm.cttz.i32(i32 undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 1 for: %I64 = call i64 @llvm.cttz.i64(i64 undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 5 for: %I128 = call i128 @llvm.cttz.i128(i128 undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of 13 for: %V2I128 = call <2 x i128> @llvm.cttz.v2i128(<2 x i128> undef, i1 false)
+; CHECK-CSSC-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %I8 = call i8 @llvm.cttz.i8(i8 undef, i1 false)
   %I16 = call i16 @llvm.cttz.i16(i16 undef, i1 false)
   %I32 = call i32 @llvm.cttz.i32(i32 undef, i1 false)
   %I64 = call i64 @llvm.cttz.i64(i64 undef, i1 false)
   %I128 = call i128 @llvm.cttz.i128(i128 undef, i1 false)
+  %V2I128 = call <2 x i128> @llvm.cttz.v2i128(<2 x i128> undef, i1 false)
   ret void
 }
 
@@ -38,10 +49,9 @@ define void @vec() {
 ; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V4I32 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found costs of 6 for: %V8I32 = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found costs of 12 for: %V16I32 = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %V2I64 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %V4I64 = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %V8I64 = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found costs of 8 for: %V2I128 = call <2 x i128> @llvm.cttz.v2i128(<2 x i128> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %V2I64 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 12 for: %V4I64 = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 24 for: %V8I64 = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %V2I8  = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> undef, i1 false)
@@ -66,10 +76,5 @@ define void @vec() {
   %V4I64 = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> undef, i1 false)
   %V8I64 = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> undef, i1 false)
 
-  %V2I128 = call <2 x i128> @llvm.cttz.v2i128(<2 x i128> undef, i1 false)
-
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-BASE: {{.*}}
-; CHECK-CSSC: {{.*}}

``````````

</details>


https://github.com/llvm/llvm-project/pull/214003
