[llvm] [AArch64] Update Cost model for CLMUL (PR #202965)
Matthew Devereau via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 06:30:37 PDT 2026
https://github.com/MDevereau created https://github.com/llvm/llvm-project/pull/202965
CLMUL now has custom backend lowering and the efficiency of its lowering is heavily dependant on available flags. This patch aligns the cost model from the generic costs to the realistic costs.
>From d83bdfb6c6aaceef33c977621543397bb58b4eb6 Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau at arm.com>
Date: Wed, 10 Jun 2026 12:48:12 +0000
Subject: [PATCH] [AArch64] Update Cost model for CLMUL
CLMUL now has custom backend lowering and the efficiency of its
lowering is heavily dependant on available flags. This patch aligns
the cost model from the generic costs to the realistic costs.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 50 +++++++++
llvm/test/Analysis/CostModel/AArch64/clmul.ll | 106 ++++++++++++++++--
2 files changed, 149 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..e44a9edcdd55d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -635,6 +635,56 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
return HistCost;
break;
}
+ case Intrinsic::clmul: {
+ auto LT = getTypeLegalizationCost(RetTy);
+
+ // PMUL v8i8/v16i8 is always available on AArch64
+ if (ST->hasNEON()) {
+ if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
+ return LT.first;
+
+ // Scalar i8 lowers through scalar/vector moves around PMUL.
+ if (TLI->getValueType(DL, RetTy, true) == MVT::i8)
+ return 4;
+ }
+
+ if (ST->hasSVE2() || ST->hasSME()) {
+ switch (LT.second.SimpleTy) {
+ case MVT::nxv16i8:
+ return LT.first;
+ case MVT::nxv8i16:
+ return LT.first * 6;
+ case MVT::nxv4i32:
+ return LT.first * 3;
+ case MVT::nxv2i64:
+ if (ST->hasSVEAES() && (ST->isSVEAvailable() || ST->hasSSVE_AES()))
+ return LT.first * 3;
+ return LT.first * 8;
+ default:
+ break;
+ }
+ }
+
+ if (ST->hasAES()) {
+ switch (LT.second.SimpleTy) {
+ case MVT::i16:
+ case MVT::i32:
+ case MVT::i64:
+ return LT.first * 2;
+ case MVT::i128:
+ return LT.first * 4;
+ case MVT::v1i64:
+ return LT.first;
+ case MVT::v2i64:
+ return LT.first * 3;
+ case MVT::v2i32:
+ return LT.first * 6;
+ default:
+ break;
+ }
+ }
+ break;
+ }
case Intrinsic::umin:
case Intrinsic::umax:
case Intrinsic::smin:
diff --git a/llvm/test/Analysis/CostModel/AArch64/clmul.ll b/llvm/test/Analysis/CostModel/AArch64/clmul.ll
index 3e5fa097169e1..325a4dd19a695 100644
--- a/llvm/test/Analysis/CostModel/AArch64/clmul.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/clmul.ll
@@ -1,17 +1,109 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,NOAES
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+aes | FileCheck %s --check-prefixes=CHECK,AES
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve | FileCheck %s --check-prefix=SVE
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve,+sve-aes | FileCheck %s --check-prefix=SVE-AES
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sme -force-streaming | FileCheck %s --check-prefix=SME-STREAMING
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sme,ssve-aes -force-streaming | FileCheck %s --check-prefix=SME-STREAMING-SSVE-AES
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve2 | FileCheck %s --check-prefix=SVE2
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve2,+sve-aes | FileCheck %s --check-prefix=SVE2AES
-define void @clmul(i128 %a128, i128 %b128, i64 %a64, i64 %b64, i32 %a32, i32 %b32, i8 %a8, i8 %b8) {
+define void @clmul(i128 %a128, i128 %b128, i64 %a64, i64 %b64, i32 %a32, i32 %b32, i16 %a16, i16 %b16, i8 %a8, i8 %b8) {
; CHECK-LABEL: 'clmul'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 768 for instruction: %call_i128 = call i128 @llvm.clmul.i128(i128 %a128, i128 %b128)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %call_i64 = call i64 @llvm.clmul.i64(i64 %a64, i64 %b64)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %call_i32 = call i32 @llvm.clmul.i32(i32 %a32, i32 %b32)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %call_i8 = call i8 @llvm.clmul.i8(i8 %a8, i8 %b8)
-; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; NOAES-NEXT: Cost Model: Found an estimated cost of 768 for instruction: %call_i128 = call i128 @llvm.clmul.i128(i128 %a128, i128 %b128)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %call_i64 = call i64 @llvm.clmul.i64(i64 %a64, i64 %b64)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %call_i32 = call i32 @llvm.clmul.i32(i32 %a32, i32 %b32)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %call_i16 = call i16 @llvm.clmul.i16(i16 %a16, i16 %b16)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %call_i8 = call i8 @llvm.clmul.i8(i8 %a8, i8 %b8)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AES-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %call_i128 = call i128 @llvm.clmul.i128(i128 %a128, i128 %b128)
+; AES-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %call_i64 = call i64 @llvm.clmul.i64(i64 %a64, i64 %b64)
+; AES-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %call_i32 = call i32 @llvm.clmul.i32(i32 %a32, i32 %b32)
+; AES-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %call_i16 = call i16 @llvm.clmul.i16(i16 %a16, i16 %b16)
+; AES-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %call_i8 = call i8 @llvm.clmul.i8(i8 %a8, i8 %b8)
+; AES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%call_i128 = call i128 @llvm.clmul.i128(i128 %a128, i128 %b128)
%call_i64 = call i64 @llvm.clmul.i64(i64 %a64, i64 %b64)
%call_i32 = call i32 @llvm.clmul.i32(i32 %a32, i32 %b32)
+ %call_i16 = call i16 @llvm.clmul.i16(i16 %a16, i16 %b16)
%call_i8 = call i8 @llvm.clmul.i8(i8 %a8, i8 %b8)
ret void
}
+
+define void @clmul_fixed(<16 x i8> %a8, <16 x i8> %b8, <8 x i8> %a8_64, <8 x i8> %b8_64, <2 x i64> %a64, <2 x i64> %b64, <1 x i64> %a1, <1 x i64> %b1, <4 x i32> %a32, <4 x i32> %b32, <2 x i32> %a2, <2 x i32> %b2, <4 x i16> %a16, <4 x i16> %b16, <8 x i16> %a16x8, <8 x i16> %b16x8) {
+; CHECK-LABEL: 'clmul_fixed'
+; NOAES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <16 x i8> @llvm.clmul.v16i8(<16 x i8> %a8, <16 x i8> %b8)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8_64 = call <8 x i8> @llvm.clmul.v8i8(<8 x i8> %a8_64, <8 x i8> %b8_64)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %c64 = call <2 x i64> @llvm.clmul.v2i64(<2 x i64> %a64, <2 x i64> %b64)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %c1 = call <1 x i64> @llvm.clmul.v1i64(<1 x i64> %a1, <1 x i64> %b1)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %c32 = call <4 x i32> @llvm.clmul.v4i32(<4 x i32> %a32, <4 x i32> %b32)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %c2 = call <2 x i32> @llvm.clmul.v2i32(<2 x i32> %a2, <2 x i32> %b2)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16 = call <4 x i16> @llvm.clmul.v4i16(<4 x i16> %a16, <4 x i16> %b16)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16x8 = call <8 x i16> @llvm.clmul.v8i16(<8 x i16> %a16x8, <8 x i16> %b16x8)
+; NOAES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <16 x i8> @llvm.clmul.v16i8(<16 x i8> %a8, <16 x i8> %b8)
+; AES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8_64 = call <8 x i8> @llvm.clmul.v8i8(<8 x i8> %a8_64, <8 x i8> %b8_64)
+; AES-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c64 = call <2 x i64> @llvm.clmul.v2i64(<2 x i64> %a64, <2 x i64> %b64)
+; AES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c1 = call <1 x i64> @llvm.clmul.v1i64(<1 x i64> %a1, <1 x i64> %b1)
+; AES-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %c32 = call <4 x i32> @llvm.clmul.v4i32(<4 x i32> %a32, <4 x i32> %b32)
+; AES-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %c2 = call <2 x i32> @llvm.clmul.v2i32(<2 x i32> %a2, <2 x i32> %b2)
+; AES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16 = call <4 x i16> @llvm.clmul.v4i16(<4 x i16> %a16, <4 x i16> %b16)
+; AES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16x8 = call <8 x i16> @llvm.clmul.v8i16(<8 x i16> %a16x8, <8 x i16> %b16x8)
+; AES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ %c8 = call <16 x i8> @llvm.clmul.v16i8(<16 x i8> %a8, <16 x i8> %b8)
+ %c8_64 = call <8 x i8> @llvm.clmul.v8i8(<8 x i8> %a8_64, <8 x i8> %b8_64)
+ %c64 = call <2 x i64> @llvm.clmul.v2i64(<2 x i64> %a64, <2 x i64> %b64)
+ %c1 = call <1 x i64> @llvm.clmul.v1i64(<1 x i64> %a1, <1 x i64> %b1)
+ %c32 = call <4 x i32> @llvm.clmul.v4i32(<4 x i32> %a32, <4 x i32> %b32)
+ %c2 = call <2 x i32> @llvm.clmul.v2i32(<2 x i32> %a2, <2 x i32> %b2)
+ %c16 = call <4 x i16> @llvm.clmul.v4i16(<4 x i16> %a16, <4 x i16> %b16)
+ %c16x8 = call <8 x i16> @llvm.clmul.v8i16(<8 x i16> %a16x8, <8 x i16> %b16x8)
+ ret void
+}
+
+define void @clmul_scalable(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8, <vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16, <vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32, <vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64) {
+; SVE-LABEL: 'clmul_scalable'
+; SVE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SVE-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SVE-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SVE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SVE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SVE-AES-LABEL: 'clmul_scalable'
+; SVE-AES-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SVE-AES-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SVE-AES-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SVE-AES-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SVE-AES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SME-STREAMING-LABEL: 'clmul_scalable'
+; SME-STREAMING-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SME-STREAMING-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SME-STREAMING-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SME-STREAMING-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SME-STREAMING-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SME-STREAMING-SSVE-AES-LABEL: 'clmul_scalable'
+; SME-STREAMING-SSVE-AES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SME-STREAMING-SSVE-AES-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SME-STREAMING-SSVE-AES-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SME-STREAMING-SSVE-AES-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SME-STREAMING-SSVE-AES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SVE2-LABEL: 'clmul_scalable'
+; SVE2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SVE2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SVE2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SVE2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SVE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SVE2AES-LABEL: 'clmul_scalable'
+; SVE2AES-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+; SVE2AES-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+; SVE2AES-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+; SVE2AES-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+; SVE2AES-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ %c8 = call <vscale x 16 x i8> @llvm.clmul.nxv16i8(<vscale x 16 x i8> %a8, <vscale x 16 x i8> %b8)
+ %c16 = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %a16, <vscale x 8 x i16> %b16)
+ %c32 = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %a32, <vscale x 4 x i32> %b32)
+ %c64 = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %a64, <vscale x 2 x i64> %b64)
+ ret void
+}
More information about the llvm-commits
mailing list