[llvm] [AArch64] Better bitcast costs. (PR #227354)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 08:48:09 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/227354
A bitcast between different register banks is essentially the same as a vector lane insert/extract, and should be costed the same as a cross-regbank copy. Between the same regbank it is free (outside of BE, which this does not try to handle, and bitcasts that need packing for which this is better than the existing fallback of 1 but isn't very accurate yet).
>From a878fb595f23003c9c70d31ba3c2d38d07fd0971 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 29 Sep 2026 16:07:44 +0100
Subject: [PATCH] [AArch64] Better bitcast costs.
A bitcast between different register banks is essentially the same as a vector lane
insert/extract, and should be costed the same as a cross-regbank copy. Between
the same regbank it is free (outside of BE, which this does not try to handle,
and bitcasts that need packing for which this is better than the existing
fallback of 1 but isn't very accurate yet).
---
.../AArch64/AArch64TargetTransformInfo.cpp | 32 ++++++++----
llvm/test/Analysis/CostModel/AArch64/cast.ll | 51 +++++++++++--------
.../Analysis/CostModel/AArch64/sve-cast.ll | 5 ++
.../AArch64/bitreverse-v8i8-to-gpr.ll | 4 +-
.../AArch64/sve-interleave-splat.ll | 7 +--
5 files changed, 63 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ddea78fa4e808..948bdd8fd389a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3960,6 +3960,28 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
EVT SrcTy = TLI->getValueType(DL, Src);
EVT DstTy = TLI->getValueType(DL, Dst);
+ // A bitcast that crosses register banks is costed as a vector insert/extract.
+ // Between the same register bank it is free. Note: Some costs are not handled
+ // well by this, like i1 vectors that need to be packed.
+ if (ISD == ISD::BITCAST && ST->isLittleEndian()) {
+ std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
+ std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
+ bool SrcIsFloatOrVec =
+ SrcLT.second.isVector() || SrcLT.second.isFloatingPoint();
+ bool DstIsFloatOrVec =
+ DstLT.second.isVector() || DstLT.second.isFloatingPoint();
+ if (SrcIsFloatOrVec == DstIsFloatOrVec &&
+ (SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits() ||
+ (SrcLT.second.isScalableVector() &&
+ SrcLT.second.getScalarSizeInBits() != 1 &&
+ DstLT.second.getScalarSizeInBits() != 1)))
+ return 0;
+ return std::max(SrcLT.first, DstLT.first) *
+ (CostKind == TTI::TCK_CodeSize
+ ? 1
+ : ST->getVectorInsertExtractBaseCost());
+ }
+
// From a vector to a scalarized vector will be an series of extract-element
// and extends.
if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
@@ -4468,16 +4490,6 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
{ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2},
{ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6},
- // Bitcasts from float to integer
- {ISD::BITCAST, MVT::nxv2f16, MVT::nxv2i16, 0},
- {ISD::BITCAST, MVT::nxv4f16, MVT::nxv4i16, 0},
- {ISD::BITCAST, MVT::nxv2f32, MVT::nxv2i32, 0},
-
- // Bitcasts from integer to float
- {ISD::BITCAST, MVT::nxv2i16, MVT::nxv2f16, 0},
- {ISD::BITCAST, MVT::nxv4i16, MVT::nxv4f16, 0},
- {ISD::BITCAST, MVT::nxv2i32, MVT::nxv2f32, 0},
-
// Add cost for extending to illegal -too wide- scalable vectors.
// zero/sign extend are implemented by multiple unpack operations,
// where each operation has a cost of 1.
diff --git a/llvm/test/Analysis/CostModel/AArch64/cast.ll b/llvm/test/Analysis/CostModel/AArch64/cast.ll
index 04f3e15ab6243..602d2f1b05bae 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cast.ll
@@ -858,30 +858,30 @@ define void @bitcasts() {
; CHECK-LABEL: 'bitcasts'
; CHECK-NEXT: Cost Model: Found costs of 0 for: %a = bitcast i32 poison to i32
; CHECK-NEXT: Cost Model: Found costs of 0 for: %b = bitcast float poison to float
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %c = bitcast i32 poison to float
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %d = bitcast float poison to i32
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %e = bitcast i64 poison to double
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %f = bitcast double poison to i64
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %g1 = bitcast half poison to i16
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %h1 = bitcast i16 poison to half
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %c = bitcast i32 poison to float
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %d = bitcast float poison to i32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %e = bitcast i64 poison to double
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %f = bitcast double poison to i64
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %g1 = bitcast half poison to i16
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %h1 = bitcast i16 poison to half
; CHECK-NEXT: Cost Model: Found costs of 0 for: %g2 = bitcast half poison to bfloat
; CHECK-NEXT: Cost Model: Found costs of 0 for: %h2 = bitcast bfloat poison to half
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %g3 = bitcast bfloat poison to i16
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %h3 = bitcast i16 poison to bfloat
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i8 = bitcast <4 x i8> poison to i32
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i8r = bitcast i32 poison to <4 x i8>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16 = bitcast <4 x i16> poison to i64
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v4i16r = bitcast i64 poison to <4 x i16>
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v4i32 = bitcast <4 x i32> poison to i128
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v4i32r = bitcast i128 poison to <4 x i32>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %g3 = bitcast bfloat poison to i16
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %h3 = bitcast i16 poison to bfloat
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i8 = bitcast <4 x i8> poison to i32
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i8r = bitcast i32 poison to <4 x i8>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i16 = bitcast <4 x i16> poison to i64
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v4i16r = bitcast i64 poison to <4 x i16>
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %v4i32 = bitcast <4 x i32> poison to i128
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %v4i32r = bitcast i128 poison to <4 x i32>
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v4i64 = bitcast <4 x i64> poison to i256
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %v4i64r = bitcast i256 poison to <4 x i64>
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i128 = bitcast <4 x i128> poison to i512
-; CHECK-NEXT: Cost Model: Found costs of 8 for: %v4i128r = bitcast i512 poison to <4 x i128>
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %v2f8 = bitcast <2 x i8> poison to half
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2f8r = bitcast half poison to <2 x i8>
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2f16 = bitcast <2 x i16> poison to float
-; CHECK-NEXT: Cost Model: Found costs of 1 for: %v2f16r = bitcast float poison to <2 x i16>
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %v4i128 = bitcast <4 x i128> poison to i512
+; CHECK-NEXT: Cost Model: Found costs of 0 for: %v4i128r = bitcast i512 poison to <4 x i128>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f8 = bitcast <2 x i8> poison to half
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f8r = bitcast half poison to <2 x i8>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f16 = bitcast <2 x i16> poison to float
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v2f16r = bitcast float poison to <2 x i16>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v2f32 = bitcast <2 x i32> poison to double
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v2f32r = bitcast double poison to <2 x i32>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %v2f64 = bitcast <2 x i64> poison to fp128
@@ -889,7 +889,11 @@ define void @bitcasts() {
; CHECK-NEXT: Cost Model: Found costs of 0 for: %vec8 = bitcast <4 x i8> poison to <2 x i16>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %vec16 = bitcast <4 x i16> poison to <2 x i32>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %vec32 = bitcast <4 x i32> poison to <2 x i64>
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:6 Lat:10 SizeLat:10 for: %vec64 = bitcast <4 x i64> poison to <2 x i128>
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %vec64 = bitcast <4 x i64> poison to <2 x i128>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %i1v2 = bitcast <16 x i1> poison to <2 x i8>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %i1v2r = bitcast <2 x i8> poison to <16 x i1>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %i1i = bitcast <16 x i1> poison to i16
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %i1ir = bitcast i16 poison to <16 x i1>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%a = bitcast i32 poison to i32
@@ -929,6 +933,11 @@ define void @bitcasts() {
%vec16 = bitcast <4 x i16> poison to <2 x i32>
%vec32 = bitcast <4 x i32> poison to <2 x i64>
%vec64 = bitcast <4 x i64> poison to <2 x i128>
+
+ %i1v2 = bitcast <16 x i1> poison to <2 x i8>
+ %i1v2r = bitcast <2 x i8> poison to <16 x i1>
+ %i1i = bitcast <16 x i1> poison to i16
+ %i1ir = bitcast i16 poison to <16 x i1>
ret void
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
index 8b9a35167b2b4..0ef5044bdaf67 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-cast.ll
@@ -1768,6 +1768,8 @@ define i32 @bitcasts() {
; CHECK-NEXT: Cost Model: Found costs of 0 for: %f = bitcast <vscale x 4 x double> poison to <vscale x 4 x i64>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %g = bitcast <vscale x 4 x half> poison to <vscale x 4 x i16>
; CHECK-NEXT: Cost Model: Found costs of 0 for: %h = bitcast <vscale x 4 x i16> poison to <vscale x 4 x half>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %v = bitcast <vscale x 16 x i1> poison to <vscale x 2 x i8>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %w = bitcast <vscale x 2 x i8> poison to <vscale x 16 x i1>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%a = bitcast <vscale x 4 x i32> poison to <vscale x 4 x i32>
@@ -1778,6 +1780,9 @@ define i32 @bitcasts() {
%f = bitcast <vscale x 4 x double> poison to <vscale x 4 x i64>
%g = bitcast <vscale x 4 x half> poison to <vscale x 4 x i16>
%h = bitcast <vscale x 4 x i16> poison to <vscale x 4 x half>
+
+ %v = bitcast <vscale x 16 x i1> poison to <vscale x 2 x i8>
+ %w = bitcast <vscale x 2 x i8> poison to <vscale x 16 x i1>
ret i32 undef
}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/bitreverse-v8i8-to-gpr.ll b/llvm/test/Transforms/VectorCombine/AArch64/bitreverse-v8i8-to-gpr.ll
index af3e6bd067db5..f751d544ab7b2 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/bitreverse-v8i8-to-gpr.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/bitreverse-v8i8-to-gpr.ll
@@ -81,8 +81,8 @@ define i64 @no_fold_innerbitcast_multiuse(i64 %x, ptr %p) {
; CHECK-SAME: i64 [[X:%.*]], ptr [[P:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[X]] to <8 x i8>
; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[P]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i8> @llvm.bitreverse.v8i8(<8 x i8> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i8> [[TMP2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[X]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.bitreverse.i64(i64 [[TMP2]])
; CHECK-NEXT: ret i64 [[TMP3]]
;
%1 = bitcast i64 %x to <8 x i8>
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/sve-interleave-splat.ll b/llvm/test/Transforms/VectorCombine/AArch64/sve-interleave-splat.ll
index 921bcf086f2bf..7e2ad90584c81 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/sve-interleave-splat.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/sve-interleave-splat.ll
@@ -1,11 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=vector-combine %s -S -o - | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
define <vscale x 4 x i16> @interleave2_same_const_splat_nxv4i16() {
-;CHECK-LABEL: @interleave2_same_const_splat_nxv4i16(
-;CHECK: call <vscale x 4 x i16> @llvm.vector.interleave2
-;CHECK: ret <vscale x 4 x i16> %retval
+; CHECK-LABEL: define <vscale x 4 x i16> @interleave2_same_const_splat_nxv4i16() {
+; CHECK-NEXT: ret <vscale x 4 x i16> bitcast (<vscale x 2 x i32> splat (i32 196611) to <vscale x 4 x i16>)
+;
%retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3))
ret <vscale x 4 x i16> %retval
}
More information about the llvm-commits
mailing list