[llvm-branch-commits] [llvm] [TTI] Add cost for @llvm.mask.beforefirst (PR #212174)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 15 03:24:05 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
Author: Luke Lau (lukel97)
<details>
<summary>Changes</summary>
Stacked on #<!-- -->203874
This implements the cost model for the proposed `@<!-- -->llvm.mask.beforefirst` intrinsic.
BasicTTIImpl returns a cost of 1 when it's legal, and a generic expansion cost otherwise. This gives the correct cost for RISC-V. For AArch64 the fixed length vectors are expanded `get.active.lane.mask` and then recombined into `brkb`, so handle that cost in `AArch64TTIImpl::getIntrinsicInstrCost`.
---
Full diff: https://github.com/llvm/llvm-project/pull/212174.diff
3 Files Affected:
- (modified) llvm/include/llvm/CodeGen/BasicTTIImpl.h (+23)
- (added) llvm/test/Analysis/CostModel/AArch64/mask-beforefirst.ll (+44)
- (added) llvm/test/Analysis/CostModel/RISCV/mask-beforefirst.ll (+42)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 0d544c602dd977..fc24cf7d68f450 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2187,6 +2187,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
case Intrinsic::get_active_lane_mask:
+ case Intrinsic::mask_beforefirst:
case Intrinsic::experimental_vector_match:
case Intrinsic::experimental_vector_histogram_add:
case Intrinsic::experimental_vector_histogram_uadd_sat:
@@ -2652,6 +2653,28 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
CmpInst::ICMP_ULT, CostKind);
return Cost;
}
+ case Intrinsic::mask_beforefirst: {
+ ISD = ISD::MASK_BEFOREFIRST;
+ std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
+ if (!getTLI()->isOperationExpand(ISD, LT.second))
+ break;
+
+ // Expansion via (get_active_lane_mask 0, (cttz.elts mask))
+ RetTy = VectorType::get(RetTy->getScalarType(),
+ LT.second.getVectorElementCount());
+ Type *IdxTy =
+ IntegerType::get(RetTy->getContext(),
+ getTLI()->getVectorIdxTy(DL).getFixedSizeInBits());
+ IntrinsicCostAttributes CttzEltsAttrs(Intrinsic::experimental_cttz_elts,
+ IdxTy, RetTy, FMF);
+ InstructionCost Cost =
+ thisT()->getIntrinsicInstrCost(CttzEltsAttrs, CostKind);
+
+ IntrinsicCostAttributes ActiveLaneMaskAttrs(
+ Intrinsic::get_active_lane_mask, RetTy, {IdxTy, IdxTy}, FMF);
+ Cost += thisT()->getIntrinsicInstrCost(ActiveLaneMaskAttrs, CostKind);
+ return Cost * LT.first;
+ }
case Intrinsic::experimental_memset_pattern:
// This cost is set to match the cost of the memset_pattern16 libcall.
// It should likely be re-evaluated after migration to this intrinsic
diff --git a/llvm/test/Analysis/CostModel/AArch64/mask-beforefirst.ll b/llvm/test/Analysis/CostModel/AArch64/mask-beforefirst.ll
new file mode 100644
index 00000000000000..48a6200141be03
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/mask-beforefirst.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve | FileCheck %s --check-prefix=SVE
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 | FileCheck %s --check-prefix=NEON
+
+define void @mask_beforefirst() {
+; SVE-LABEL: 'mask_beforefirst'
+; SVE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = call <vscale x 2 x i1> @llvm.mask.beforefirst.nxv2i1(<vscale x 2 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = call <vscale x 16 x i1> @llvm.mask.beforefirst.nxv16i1(<vscale x 16 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = call <vscale x 32 x i1> @llvm.mask.beforefirst.nxv32i1(<vscale x 32 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call <2 x i1> @llvm.mask.beforefirst.v2i1(<2 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %8 = call <8 x i1> @llvm.mask.beforefirst.v8i1(<8 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = call <16 x i1> @llvm.mask.beforefirst.v16i1(<16 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = call <32 x i1> @llvm.mask.beforefirst.v32i1(<32 x i1> poison)
+; SVE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; NEON-LABEL: 'mask_beforefirst'
+; NEON-NEXT: Cost Model: Invalid cost for instruction: %1 = call <vscale x 2 x i1> @llvm.mask.beforefirst.nxv2i1(<vscale x 2 x i1> poison)
+; NEON-NEXT: Cost Model: Invalid cost for instruction: %2 = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> poison)
+; NEON-NEXT: Cost Model: Invalid cost for instruction: %3 = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> poison)
+; NEON-NEXT: Cost Model: Invalid cost for instruction: %4 = call <vscale x 16 x i1> @llvm.mask.beforefirst.nxv16i1(<vscale x 16 x i1> poison)
+; NEON-NEXT: Cost Model: Invalid cost for instruction: %5 = call <vscale x 32 x i1> @llvm.mask.beforefirst.nxv32i1(<vscale x 32 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %6 = call <2 x i1> @llvm.mask.beforefirst.v2i1(<2 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %7 = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %8 = call <8 x i1> @llvm.mask.beforefirst.v8i1(<8 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %9 = call <16 x i1> @llvm.mask.beforefirst.v16i1(<16 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 160 for instruction: %10 = call <32 x i1> @llvm.mask.beforefirst.v32i1(<32 x i1> poison)
+; NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call <vscale x 2 x i1> @llvm.mask.beforefirst(<vscale x 2 x i1> poison)
+ call <vscale x 4 x i1> @llvm.mask.beforefirst(<vscale x 4 x i1> poison)
+ call <vscale x 8 x i1> @llvm.mask.beforefirst(<vscale x 8 x i1> poison)
+ call <vscale x 16 x i1> @llvm.mask.beforefirst(<vscale x 16 x i1> poison)
+ call <vscale x 32 x i1> @llvm.mask.beforefirst(<vscale x 32 x i1> poison)
+ call <2 x i1> @llvm.mask.beforefirst(<2 x i1> poison)
+ call <4 x i1> @llvm.mask.beforefirst(<4 x i1> poison)
+ call <8 x i1> @llvm.mask.beforefirst(<8 x i1> poison)
+ call <16 x i1> @llvm.mask.beforefirst(<16 x i1> poison)
+ call <32 x i1> @llvm.mask.beforefirst(<32 x i1> poison)
+
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/RISCV/mask-beforefirst.ll b/llvm/test/Analysis/CostModel/RISCV/mask-beforefirst.ll
new file mode 100644
index 00000000000000..4c71dd6bf5a513
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/mask-beforefirst.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes='print<cost-model>' 2>&1 -disable-output -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+define void @mask_beforefirst() {
+; CHECK-LABEL: 'mask_beforefirst'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = call <vscale x 1 x i1> @llvm.mask.beforefirst.nxv1i1(<vscale x 1 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = call <vscale x 2 x i1> @llvm.mask.beforefirst.nxv2i1(<vscale x 2 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = call <vscale x 16 x i1> @llvm.mask.beforefirst.nxv16i1(<vscale x 16 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 32 x i1> @llvm.mask.beforefirst.nxv32i1(<vscale x 32 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 64 x i1> @llvm.mask.beforefirst.nxv64i1(<vscale x 64 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = call <vscale x 128 x i1> @llvm.mask.beforefirst.nxv128i1(<vscale x 128 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %9 = call <1 x i1> @llvm.mask.beforefirst.v1i1(<1 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = call <2 x i1> @llvm.mask.beforefirst.v2i1(<2 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = call <8 x i1> @llvm.mask.beforefirst.v8i1(<8 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %13 = call <16 x i1> @llvm.mask.beforefirst.v16i1(<16 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = call <32 x i1> @llvm.mask.beforefirst.v32i1(<32 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = call <64 x i1> @llvm.mask.beforefirst.v64i1(<64 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = call <128 x i1> @llvm.mask.beforefirst.v128i1(<128 x i1> poison)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call <vscale x 1 x i1> @llvm.mask.beforefirst(<vscale x 1 x i1> poison)
+ call <vscale x 2 x i1> @llvm.mask.beforefirst(<vscale x 2 x i1> poison)
+ call <vscale x 4 x i1> @llvm.mask.beforefirst(<vscale x 4 x i1> poison)
+ call <vscale x 8 x i1> @llvm.mask.beforefirst(<vscale x 8 x i1> poison)
+ call <vscale x 16 x i1> @llvm.mask.beforefirst(<vscale x 16 x i1> poison)
+ call <vscale x 32 x i1> @llvm.mask.beforefirst(<vscale x 32 x i1> poison)
+ call <vscale x 64 x i1> @llvm.mask.beforefirst(<vscale x 64 x i1> poison)
+ call <vscale x 128 x i1> @llvm.mask.beforefirst(<vscale x 128 x i1> poison)
+ call <1 x i1> @llvm.mask.beforefirst(<1 x i1> poison)
+ call <2 x i1> @llvm.mask.beforefirst(<2 x i1> poison)
+ call <4 x i1> @llvm.mask.beforefirst(<4 x i1> poison)
+ call <8 x i1> @llvm.mask.beforefirst(<8 x i1> poison)
+ call <16 x i1> @llvm.mask.beforefirst(<16 x i1> poison)
+ call <32 x i1> @llvm.mask.beforefirst(<32 x i1> poison)
+ call <64 x i1> @llvm.mask.beforefirst(<64 x i1> poison)
+ call <128 x i1> @llvm.mask.beforefirst(<128 x i1> poison)
+
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/212174
More information about the llvm-branch-commits
mailing list