[llvm-branch-commits] [llvm] [AArch64][SME] Set inline incompatibility for SME using cost-model. (PR #233844)
Sander de Smalen via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 22 06:58:37 PDT 2026
https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/233844
>From 706e4467de1a35370534cc1dac0553f47acca84d Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Tue, 6 Oct 2026 16:05:35 +0000
Subject: [PATCH 1/2] [AArch64][SME] Set inline incompatibility for SME using
cost-model.
Instead of maintaining a new switch-statement where we handle
individual intrinsics, rely on the TTI cost-model to check if
an intrinsic can be inlined into a streaming function.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 26 ++---
.../Inline/AArch64/sme-pstatesm-attrs.ll | 100 +++++++++++++++++-
2 files changed, 112 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 48ea8d18dd83d4..ad0415a6455390 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -218,6 +218,15 @@ bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F,
if (!ConsiderSM)
continue;
+ // If the operation is invalid in the current mode, it should not be
+ // inlined.
+ SmallVector<Value *> Opnds(I.op_begin(), I.op_end());
+ if (!getInstructionCost(&I, Opnds, TTI::TCK_RecipThroughput).isValid())
+ return true;
+
+ if (AssumeVScaleIsEquivalent)
+ continue;
+
// Inlining operations on fixed-length vectors when the streaming
// mode does not match is rejected because performance may be impacted.
// This decision should eventually be moved to the cost-model.
@@ -233,19 +242,10 @@ bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F,
default:
if (!Intrinsic::isTargetIntrinsic(IID))
break;
- // Conservatively assume target intrinsics are vscale dependent
- // or invalid in the current mode.
+ // Conservatively assume target intrinsics are vscale dependent,
+ // although we can probably limit it to a small list of target
+ // intrinsics that use vscale (e.g. `llvm.aarch64.sve.cntb()`).
return true;
- case Intrinsic::masked_gather:
- case Intrinsic::masked_scatter:
- case Intrinsic::masked_compressstore:
- case Intrinsic::experimental_vector_histogram_add:
- // Instructions that are not available in streaming mode.
- // Fixed-length operations can still be code-generated by scalarizing.
- // FIXME: Consider FEAT_FA64.
- if (I.getOperand(0)->getType()->isScalableTy())
- return true;
- break;
case Intrinsic::vscale:
if (!AssumeVScaleIsEquivalent)
return true;
@@ -639,7 +639,7 @@ static InstructionCost getHistogramCost(const AArch64Subtarget *ST,
return InstructionCost::getInvalid();
// Only interested in costing for the hardware instruction from SVE2.
- if (!ST->hasSVE2())
+ if (!ST->isSVEAvailable() || !ST->hasSVE2())
return InstructionCost::getInvalid();
Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers
diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
index ddd625e47cf880..3c47d7da772f74 100644
--- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
+++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
@@ -676,7 +676,7 @@ define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_st
define void @invoke_opaque_fptr(ptr %fptr) alwaysinline personality ptr null {
; CHECK-LABEL: define void @invoke_opaque_fptr
; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR6]] personality ptr null {
-; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR15:[0-9]+]]
+; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR24:[0-9]+]]
; CHECK-NEXT: to label [[NORMAL_RETURN:%.*]] unwind label [[UNWIND_CLEANUP:%.*]]
; CHECK: normal_return:
; CHECK-NEXT: ret void
@@ -1098,6 +1098,104 @@ define <vscale x 2 x i64> @gather_scalable_length_caller(<vscale x 2 x ptr> %ptr
ret <vscale x 2 x i64> %res
}
+;
+; Test that we can inline a compressstore into a streaming function with +sme2p2,
+; but not without +sme2p2.
+;
+
+define void @sme2p2_compress_store_scalable(<vscale x 2 x i64> %v, ptr %p) "target-features"="+sve,+sme2p2" {
+; CHECK-LABEL: define void @sme2p2_compress_store_scalable
+; CHECK-SAME: (<vscale x 2 x i64> [[V:%.*]], ptr [[P:%.*]]) #[[ATTR10:[0-9]+]] {
+; CHECK-NEXT: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> [[V]], ptr [[P]], <vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.compressstore(<vscale x 2 x i64> %v, ptr %p, <vscale x 2 x i1> splat(i1 true))
+ ret void
+}
+define void @sme2p2_compress_store_scalable_caller(<vscale x 2 x i64> %v, ptr %p) "aarch64_pstate_sm_enabled" "target-features"="+sve,+sme2p2" {
+; CHECK-LABEL: define void @sme2p2_compress_store_scalable_caller
+; CHECK-SAME: (<vscale x 2 x i64> [[V:%.*]], ptr [[P:%.*]]) #[[ATTR11:[0-9]+]] {
+; CHECK-NEXT: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> [[V]], ptr [[P]], <vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @sme2p2_compress_store_scalable(<vscale x 2 x i64> %v, ptr %p)
+ ret void
+}
+
+define void @sme_compress_store_scalable(<vscale x 2 x i64> %v, ptr %p) "target-features"="+sve,+sme" {
+; CHECK-LABEL: define void @sme_compress_store_scalable
+; CHECK-SAME: (<vscale x 2 x i64> [[V:%.*]], ptr [[P:%.*]]) #[[ATTR12:[0-9]+]] {
+; CHECK-NEXT: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> [[V]], ptr [[P]], <vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.compressstore(<vscale x 2 x i64> %v, ptr %p, <vscale x 2 x i1> splat(i1 true))
+ ret void
+}
+
+; Test the compressstore is *not* inlined, as the caller doesn't have +sme2p2.
+define void @sme_compress_store_scalable_caller(<vscale x 2 x i64> %v, ptr %p) "aarch64_pstate_sm_enabled" "target-features"="+sve,+sme" {
+; CHECK-LABEL: define void @sme_compress_store_scalable_caller
+; CHECK-SAME: (<vscale x 2 x i64> [[V:%.*]], ptr [[P:%.*]]) #[[ATTR13:[0-9]+]] {
+; CHECK-NEXT: call void @sme_compress_store_scalable(<vscale x 2 x i64> [[V]], ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call void @sme_compress_store_scalable(<vscale x 2 x i64> %v, ptr %p)
+ ret void
+}
+
+;
+; Test that histogram.add is not inlined, as HISTCNT not available in streaming mode.
+;
+
+define void @sme2_sve2_histcnt(<vscale x 2 x ptr> %ptrs, i64 %inc) "target-features"="+sve2,+sme2" {
+; CHECK-LABEL: define void @sme2_sve2_histcnt
+; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], i64 [[INC:%.*]]) #[[ATTR14:[0-9]+]] {
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> [[PTRS]], i64 [[INC]], <vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @llvm.experimental.vector.histogram.add(<vscale x 2 x ptr> %ptrs, i64 %inc, <vscale x 2 x i1> splat(i1 true))
+ ret void
+}
+
+define void @sme2_sve2_histcnt_caller(<vscale x 2 x ptr> %ptrs, i64 %inc) "aarch64_pstate_sm_enabled" "target-features"="+sve2,+sme2" {
+; CHECK-LABEL: define void @sme2_sve2_histcnt_caller
+; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], i64 [[INC:%.*]]) #[[ATTR15:[0-9]+]] {
+; CHECK-NEXT: call void @sme2_sve2_histcnt(<vscale x 2 x ptr> [[PTRS]], i64 [[INC]])
+; CHECK-NEXT: ret void
+;
+ call void @sme2_sve2_histcnt(<vscale x 2 x ptr> %ptrs, i64 %inc)
+ ret void
+}
+
+;
+; Test that vector.match can be inlined.
+
+; Even thought MATCH is not available in streaming mode, it can be broken down
+; into a series of vector compares for each of the elements in the 'needle' vector,
+; see TargetLowering::expandVectorMatch.
+define <vscale x 4 x i1> @sme2_sve2_match(ptr %dataptr, <4 x i32> %needles) alwaysinline "target-features"="+sve2,+sme2" {
+; CHECK-LABEL: define <vscale x 4 x i1> @sme2_sve2_match
+; CHECK-SAME: (ptr [[DATAPTR:%.*]], <4 x i32> [[NEEDLES:%.*]]) #[[ATTR16:[0-9]+]] {
+; CHECK-NEXT: [[DATA:%.*]] = load <vscale x 4 x i32>, ptr [[DATAPTR]], align 16
+; CHECK-NEXT: [[MATCH:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> [[DATA]], <4 x i32> [[NEEDLES]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: ret <vscale x 4 x i1> [[MATCH]]
+;
+ %data = load <vscale x 4 x i32>, ptr %dataptr
+ %match = call <vscale x 4 x i1> @llvm.experimental.vector.match(<vscale x 4 x i32> %data, <4 x i32> %needles, <vscale x 4 x i1> splat(i1 true))
+ ret <vscale x 4 x i1> %match
+}
+
+define <vscale x 4 x i1> @sme2_sve2_match_caller(ptr %dataptr, <4 x i32> %needles) "aarch64_pstate_sm_enabled" "target-features"="+sve2,+sme2" {
+; CHECK-LABEL: define <vscale x 4 x i1> @sme2_sve2_match_caller
+; CHECK-SAME: (ptr [[DATAPTR:%.*]], <4 x i32> [[NEEDLES:%.*]]) #[[ATTR15]] {
+; CHECK-NEXT: [[DATA_I:%.*]] = load <vscale x 4 x i32>, ptr [[DATAPTR]], align 16
+; CHECK-NEXT: [[MATCH_I:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> [[DATA_I]], <4 x i32> [[NEEDLES]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: ret <vscale x 4 x i1> [[MATCH_I]]
+;
+ %match = call <vscale x 4 x i1> @sme2_sve2_match(ptr %dataptr, <4 x i32> %needles)
+ ret <vscale x 4 x i1> %match
+}
+
;
; Test that we inline a function with a call to llvm.vscale() (which takes/returns
; no scalable vectors) when the function has vl-dependent arguments. The case for
>From 9dfd357f484329d09d6cc52765825a7b3af131ac Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Wed, 21 Oct 2026 16:20:54 +0000
Subject: [PATCH 2/2] Fixups
* Handle target intrinsics conservatively
* Added RUN line to CostModel/AArch64/histograms.ll
---
.../AArch64/AArch64TargetTransformInfo.cpp | 11 +++++++--
.../Analysis/CostModel/AArch64/histograms.ll | 3 +++
.../Inline/AArch64/sme-pstatesm-attrs.ll | 24 ++++++++++++++++---
3 files changed, 33 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ad0415a6455390..0b6f3f4e0c2287 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -218,10 +218,17 @@ bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F,
if (!ConsiderSM)
continue;
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+
// If the operation is invalid in the current mode, it should not be
// inlined.
+ // Target intrinics may not be handled by `getInstructionCost`, so
+ // remain conservative that they may not be compatible in the mode
+ // of the caller.
+ // FIXME: Be less conservative about target intrinsics.
SmallVector<Value *> Opnds(I.op_begin(), I.op_end());
- if (!getInstructionCost(&I, Opnds, TTI::TCK_RecipThroughput).isValid())
+ if (!getInstructionCost(&I, Opnds, TTI::TCK_RecipThroughput).isValid() ||
+ (II && Intrinsic::isTargetIntrinsic(II->getIntrinsicID())))
return true;
if (AssumeVScaleIsEquivalent)
@@ -236,7 +243,7 @@ bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F,
})))
return true;
- if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+ if (II) {
unsigned IID = II->getIntrinsicID();
switch (IID) {
default:
diff --git a/llvm/test/Analysis/CostModel/AArch64/histograms.ll b/llvm/test/Analysis/CostModel/AArch64/histograms.ll
index f03adb64fbe924..7f48620879393e 100644
--- a/llvm/test/Analysis/CostModel/AArch64/histograms.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/histograms.ll
@@ -3,6 +3,9 @@
; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-SVE
; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve2 | FileCheck %s --check-prefix=CHECK-SVE2
+; HISTCNT is unsupported in streaming mode, so should behave similar to CHECK-SVE.
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve2,+sme -force-streaming | FileCheck %s --check-prefix=CHECK-SVE
+
define void @histograms() {
; CHECK-NEON-LABEL: 'histograms'
; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv1p0.i64(<vscale x 1 x ptr> poison, i64 1, <vscale x 1 x i1> poison)
diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
index 3c47d7da772f74..6c62c1c666b521 100644
--- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
+++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
@@ -539,15 +539,33 @@ entry:
ret i64 %res
}
+; FIXME: This function should be inlined, but we're currently cautious around
+; inlining target intrinsics.
+define i64 @normal_callee_with_intrinsic_call_alwaysinline(<vscale x 4 x i32> %unused) alwaysinline {
+; CHECK-LABEL: define i64 @normal_callee_with_intrinsic_call_alwaysinline
+; CHECK-SAME: (<vscale x 4 x i32> [[UNUSED:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.aarch64.sve.cntb(i32 4)
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ %res = call i64 @llvm.aarch64.sve.cntb(i32 4)
+ ret i64 %res
+}
+
define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[RES:%.*]] = call i64 @normal_callee_with_intrinsic_call()
+; CHECK-NEXT: [[RES1:%.*]] = call i64 @normal_callee_with_intrinsic_call()
+; CHECK-NEXT: [[RES2:%.*]] = call i64 @normal_callee_with_intrinsic_call_alwaysinline(<vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT: [[RES:%.*]] = add i64 [[RES1]], [[RES2]]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
- %res = call i64 @normal_callee_with_intrinsic_call()
+ %res1 = call i64 @normal_callee_with_intrinsic_call()
+ %res2 = call i64 @normal_callee_with_intrinsic_call_alwaysinline(<vscale x 4 x i32> zeroinitializer)
+ %res = add i64 %res1, %res2
ret i64 %res
}
@@ -654,7 +672,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompat
define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() alwaysinline {
; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline
-; CHECK-SAME: () #[[ATTR6:[0-9]+]] {
+; CHECK-SAME: () #[[ATTR6]] {
; CHECK-NEXT: call void @streaming_compatible_body()
; CHECK-NEXT: ret void
;
More information about the llvm-branch-commits
mailing list