[llvm] [AArch64][CostModel] Add cost model for pdep/pext instructions (PR #207657)
Yashwant Singh via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 23:46:40 PDT 2026
https://github.com/yashssh updated https://github.com/llvm/llvm-project/pull/207657
>From 45bf13ed8907e715be53d2dbb11eb372054776f6 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Sun, 5 Jul 2026 22:41:15 -0700
Subject: [PATCH 1/5] [AArch64][CostModel] Add cost model for pdep/pext
instructions
---
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
llvm/lib/Target/AArch64/AArch64Subtarget.h | 8 ++++
.../AArch64/AArch64TargetTransformInfo.cpp | 23 +++++++++
.../Analysis/CostModel/AArch64/pdep-pext.ll | 47 +++++++++++++++++++
4 files changed, 79 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..8c69793bb4ce7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2176,10 +2176,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
// Map generic PEXT/PDEP to SVE2 bitperm BEXT/BDEP instructions.
- if (Subtarget->hasSVEBitPerm() &&
- (Subtarget->isSVEAvailable() ||
- (Subtarget->isSVEorStreamingSVEAvailable() &&
- Subtarget->hasSSVE_BitPerm()))) {
+ if (Subtarget->isSVEBitPermAvailable()) {
for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
setOperationAction({ISD::PEXT, ISD::PDEP}, VT, Custom);
}
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 380c3e11fcbf2..b42ea897786da 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -236,6 +236,14 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
return isSVEAvailable() || (isSVEorStreamingSVEAvailable() && hasSME2());
}
+ /// Returns true if SVE bit-permute instructions (bext/bdep/bgrp) can be
+ /// emitted.
+ bool isSVEBitPermAvailable() const {
+ return hasSVEBitPerm() &&
+ (isSVEAvailable() ||
+ (isSVEorStreamingSVEAvailable() && hasSSVE_BitPerm()));
+ }
+
unsigned getMinVectorRegisterBitWidth() const {
// Don't assume any minimum vector size when PSTATE.SM may not be 0, because
// we don't yet support streaming-compatible codegen support that we trust
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7b1c603d4134c..f6ba0fde21012 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -864,6 +864,29 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
}
break;
}
+ case Intrinsic::pdep:
+ case Intrinsic::pext: {
+ unsigned BW = RetTy->getScalarSizeInBits();
+ if (ST->isSVEBitPermAvailable() && BW <= 64) {
+ Type *VecTy = VectorType::get(RetTy, 64 / BW, false);
+ InstructionCost Cost =
+ getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
+ nullptr, nullptr) +
+ getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
+ nullptr, nullptr) +
+ 1 +
+ getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind, 0,
+ nullptr, nullptr);
+ return Cost;
+ }
+ // Without bit-permute, pdep/pext expand to the Hacker's Delight scalar
+ // sequence: O(log2(BW)) parallel-prefix stages, each a CLMUL-by-all-ones
+ // (~log2(BW) shift-xors) plus a handful of scalar ALU ops. That is
+ // O(log2(BW)^2) work; cost ~ LogBW * (LogBW + 7) matches the emitted
+ // sequence (~48/61/79 insns for i16/i32/i64).
+ unsigned LogBW = Log2_32_Ceil(BW);
+ return getTypeLegalizationCost(RetTy).first * LogBW * (LogBW + 7);
+ }
case Intrinsic::ctpop: {
auto LT = getTypeLegalizationCost(RetTy);
MVT MTy = LT.second;
diff --git a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
new file mode 100644
index 0000000000000..1960a2d1514b7
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK-NO-BITPERM
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve2-bitperm | FileCheck %s --check-prefixes=CHECK-BITPERM
+
+define void @pdep() {
+; CHECK-NO-BITPERM-LABEL: 'pdep'
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-BITPERM-LABEL: 'pdep'
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+ %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+ %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+ %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+ ret void
+}
+
+define void @pext() {
+; CHECK-NO-BITPERM-LABEL: 'pext'
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-BITPERM-LABEL: 'pext'
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+ %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+ %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+ %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+ ret void
+}
>From f4db79ba55099ece202aae25ef1986da6a257086 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Wed, 8 Jul 2026 00:10:20 -0700
Subject: [PATCH 2/5] Address review comments
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 6 ++++++
.../AArch64/AArch64TargetTransformInfo.cpp | 10 ++--------
.../test/Analysis/CostModel/AArch64/pdep-pext.ll | 16 ++++++++--------
3 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..c5d487a68d434 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2759,6 +2759,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::bitreverse:
ISD = ISD::BITREVERSE;
break;
+ case Intrinsic::pdep:
+ case Intrinsic::pext:
+ ISD = IID == Intrinsic::pdep ? ISD::PDEP : ISD::PEXT;
+ // When not legal/custom, pdep/pext expand to a long inline sequence
+ SingleCallCost = 20;
+ break;
case Intrinsic::ucmp:
ISD = ISD::UCMP;
break;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index f6ba0fde21012..ee0e3b1894829 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -867,7 +867,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
case Intrinsic::pdep:
case Intrinsic::pext: {
unsigned BW = RetTy->getScalarSizeInBits();
- if (ST->isSVEBitPermAvailable() && BW <= 64) {
+ if (!RetTy->isVectorTy() && ST->isSVEBitPermAvailable() && BW <= 64) {
Type *VecTy = VectorType::get(RetTy, 64 / BW, false);
InstructionCost Cost =
getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
@@ -879,13 +879,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
nullptr, nullptr);
return Cost;
}
- // Without bit-permute, pdep/pext expand to the Hacker's Delight scalar
- // sequence: O(log2(BW)) parallel-prefix stages, each a CLMUL-by-all-ones
- // (~log2(BW) shift-xors) plus a handful of scalar ALU ops. That is
- // O(log2(BW)^2) work; cost ~ LogBW * (LogBW + 7) matches the emitted
- // sequence (~48/61/79 insns for i16/i32/i64).
- unsigned LogBW = Log2_32_Ceil(BW);
- return getTypeLegalizationCost(RetTy).first * LogBW * (LogBW + 7);
+ break;
}
case Intrinsic::ctpop: {
auto LT = getTypeLegalizationCost(RetTy);
diff --git a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
index 1960a2d1514b7..90d38dfc698d7 100644
--- a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
@@ -4,10 +4,10 @@
define void @pdep() {
; CHECK-NO-BITPERM-LABEL: 'pdep'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-BITPERM-LABEL: 'pdep'
@@ -26,10 +26,10 @@ define void @pdep() {
define void @pext() {
; CHECK-NO-BITPERM-LABEL: 'pext'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 78 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-BITPERM-LABEL: 'pext'
>From 86a5affc7cda5d252aa43c497cbc4019a253c545 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Mon, 20 Jul 2026 02:12:27 -0700
Subject: [PATCH 3/5] Address review comments(generic cost on clmul expansion)
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 32 ++++++++++--
.../Analysis/CostModel/AArch64/pdep-pext.ll | 52 ++++++++++++++++---
.../CostModel/X86/intrinsic-cost-kinds.ll | 32 ++++++------
3 files changed, 89 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index c5d487a68d434..b1919c3805594 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2760,10 +2760,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
ISD = ISD::BITREVERSE;
break;
case Intrinsic::pdep:
+ ISD = ISD::PDEP;
+ break;
case Intrinsic::pext:
- ISD = IID == Intrinsic::pdep ? ISD::PDEP : ISD::PEXT;
- // When not legal/custom, pdep/pext expand to a long inline sequence
- SingleCallCost = 20;
+ ISD = ISD::PEXT;
break;
case Intrinsic::ucmp:
ISD = ISD::UCMP;
@@ -3160,6 +3160,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
InstructionCost PerBitCost = std::min(PerBitCostMul, PerBitCostBittest);
return BW * PerBitCost;
}
+ case Intrinsic::pdep:
+ case Intrinsic::pext: {
+ if (RetTy->isVectorTy())
+ break;
+ // Based on Hacker's Delight ยง7-5: Expand, or Generalized Insert.
+ // Below cost is accurate for pext, pdep will be similar.
+ unsigned BW = RetTy->getScalarSizeInBits();
+ InstructionCost AndCost =
+ thisT()->getArithmeticInstrCost(Instruction::And, RetTy, CostKind);
+ InstructionCost ShiftCost =
+ thisT()->getArithmeticInstrCost(Instruction::Shl, RetTy, CostKind);
+ InstructionCost OrCost =
+ thisT()->getArithmeticInstrCost(Instruction::Or, RetTy, CostKind);
+ InstructionCost XorCost =
+ thisT()->getArithmeticInstrCost(Instruction::Xor, RetTy, CostKind);
+ IntrinsicCostAttributes ClmulAttrs(Intrinsic::clmul, RetTy,
+ {RetTy, RetTy});
+ InstructionCost MulCost =
+ thisT()->getIntrinsicInstrCost(ClmulAttrs, CostKind);
+
+ int Iterations = Log2_32_Ceil(BW);
+ InstructionCost Cost = AndCost + ShiftCost; // Prefix cost
+ Cost += Iterations *
+ (MulCost + 2 * AndCost + 2 * ShiftCost + OrCost + 2 * XorCost);
+ return Cost;
+ }
default:
break;
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
index 90d38dfc698d7..fd7455bd19f73 100644
--- a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
@@ -4,10 +4,16 @@
define void @pdep() {
; CHECK-NO-BITPERM-LABEL: 'pdep'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 222 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 302 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1016 for instruction: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 608 for instruction: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-BITPERM-LABEL: 'pdep'
@@ -15,21 +21,39 @@ define void @pdep() {
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
%i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
%i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
%i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+ %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+ %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+ %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+ %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+ %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+ %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
ret void
}
define void @pext() {
; CHECK-NO-BITPERM-LABEL: 'pext'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 222 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 302 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1016 for instruction: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 608 for instruction: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; CHECK-BITPERM-LABEL: 'pext'
@@ -37,11 +61,23 @@ define void @pext() {
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
%i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
%i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
%i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+ %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+ %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+ %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+ %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+ %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+ %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
ret void
}
diff --git a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
index 45e19b93e41cf..30094e78a21d3 100644
--- a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
@@ -310,23 +310,23 @@ define void @fshl(i32 %a, i32 %b, i32 %c, <16 x i32> %va, <16 x i32> %vb, <16 x
define void @pdep(i32 %a, i32 %b, <16 x i32> %va, <16 x i32> %vb) {
; THRU-LABEL: 'pdep'
-; THRU-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
-; THRU-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; THRU-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
+; THRU-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
; THRU-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; LATE-LABEL: 'pdep'
-; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
-; LATE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; LATE-NEXT: Cost Model: Found an estimated cost of 492 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
+; LATE-NEXT: Cost Model: Found an estimated cost of 7956 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; SIZE-LABEL: 'pdep'
-; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
-; SIZE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; SIZE-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
+; SIZE-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; SIZE_LATE-LABEL: 'pdep'
-; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
-; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
+; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pdep.v16i32(<16 x i32> %va, <16 x i32> %vb)
; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
%s = call i32 @llvm.pdep.i32(i32 %a, i32 %b)
@@ -336,23 +336,23 @@ define void @pdep(i32 %a, i32 %b, <16 x i32> %va, <16 x i32> %vb) {
define void @pext(i32 %a, i32 %b, <16 x i32> %va, <16 x i32> %vb) {
; THRU-LABEL: 'pext'
-; THRU-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
-; THRU-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; THRU-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
+; THRU-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
; THRU-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; LATE-LABEL: 'pext'
-; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
-; LATE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; LATE-NEXT: Cost Model: Found an estimated cost of 492 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
+; LATE-NEXT: Cost Model: Found an estimated cost of 7956 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
; LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; SIZE-LABEL: 'pext'
-; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
-; SIZE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; SIZE-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
+; SIZE-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; SIZE_LATE-LABEL: 'pext'
-; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
-; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
+; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
+; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 4116 for instruction: %v = call <16 x i32> @llvm.pext.v16i32(<16 x i32> %va, <16 x i32> %vb)
; SIZE_LATE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
%s = call i32 @llvm.pext.i32(i32 %a, i32 %b)
>From 0ffcde77a1ac7bd3ed79875b0d0bfdf0b2d9b6b3 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Fri, 31 Jul 2026 02:50:14 -0700
Subject: [PATCH 4/5] Address review comments
- i8 and i128 tests
- pdep/pext tests for x86 with +bmi2
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 7 +-
.../Analysis/CostModel/AArch64/pdep-pext.ll | 114 +++++++++++-------
llvm/test/Analysis/CostModel/X86/pdep-pext.ll | 81 +++++++++++++
3 files changed, 154 insertions(+), 48 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/X86/pdep-pext.ll
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index b1919c3805594..10c38d7212405 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2822,7 +2822,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
const TargetLoweringBase *TLI = getTLI();
- if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
+ // pdep/pext are not lane-separable
+ bool BitManipSplit = (ISD == ISD::PDEP || ISD == ISD::PEXT) && LT.first > 1;
+
+ if (!BitManipSplit && TLI->isOperationLegalOrPromote(ISD, LT.second)) {
if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() &&
TLI->isFAbsFree(LT.second)) {
return 0;
@@ -2836,7 +2839,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return (LT.first * 2);
else
return (LT.first * 1);
- } else if (TLI->isOperationCustom(ISD, LT.second)) {
+ } else if (!BitManipSplit && TLI->isOperationCustom(ISD, LT.second)) {
// If the operation is custom lowered then assume
// that the code is twice as expensive.
return (LT.first * 2);
diff --git a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
index fd7455bd19f73..085a3d3a2c1ab 100644
--- a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
@@ -1,41 +1,54 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK-NO-BITPERM
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve2-bitperm | FileCheck %s --check-prefixes=CHECK-BITPERM
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK-NO-BITPERM
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64-- -mattr=+sve2-bitperm | FileCheck %s --check-prefixes=CHECK-BITPERM
+
+; Scalars expand to the Hacker's Delight sequence (or bdep/bext with
+; +sve2-bitperm). There is no vector pdep/pext: fixed-width vectors scalarize,
+; and scalable vectors use the native bdep/bext when available (else Invalid).
define void @pdep() {
; CHECK-NO-BITPERM-LABEL: 'pdep'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 222 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 302 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1016 for instruction: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 608 for instruction: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:44 CodeSize:35 Lat:44 SizeLat:44 for: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 222 for: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 252 for: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 302 for: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 5478 for: %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:736 CodeSize:576 Lat:736 SizeLat:736 for: %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1784 Lat:1792 SizeLat:1792 for: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:1016 CodeSize:1012 Lat:1016 SizeLat:1016 for: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:608 CodeSize:606 Lat:608 SizeLat:608 for: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv16i8 = call <vscale x 16 x i8> @llvm.pdep.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-BITPERM-LABEL: 'pdep'
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 5478 for: %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:144 CodeSize:80 Lat:144 SizeLat:144 for: %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:72 CodeSize:40 Lat:72 SizeLat:72 for: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:36 CodeSize:20 Lat:36 SizeLat:36 for: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:18 CodeSize:10 Lat:18 SizeLat:18 for: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.pdep.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
%i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
%i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
%i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+ %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+ %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
%v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
%v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
%v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+ %nxv16i8 = call <vscale x 16 x i8> @llvm.pdep.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
%nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
%nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
%nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
@@ -44,38 +57,47 @@ define void @pdep() {
define void @pext() {
; CHECK-NO-BITPERM-LABEL: 'pext'
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 222 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 252 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 302 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 1016 for instruction: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 608 for instruction: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Invalid cost for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
-; CHECK-NO-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:44 CodeSize:35 Lat:44 SizeLat:44 for: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 222 for: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 252 for: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 302 for: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of 5478 for: %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:736 CodeSize:576 Lat:736 SizeLat:736 for: %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1784 Lat:1792 SizeLat:1792 for: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:1016 CodeSize:1012 Lat:1016 SizeLat:1016 for: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:608 CodeSize:606 Lat:608 SizeLat:608 for: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv16i8 = call <vscale x 16 x i8> @llvm.pext.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-BITPERM-LABEL: 'pext'
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
-; CHECK-BITPERM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:7 SizeLat:7 for: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 5478 for: %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:144 CodeSize:80 Lat:144 SizeLat:144 for: %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:72 CodeSize:40 Lat:72 SizeLat:72 for: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:36 CodeSize:20 Lat:36 SizeLat:36 for: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:18 CodeSize:10 Lat:18 SizeLat:18 for: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv16i8 = call <vscale x 16 x i8> @llvm.pext.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
%i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
%i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
%i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+ %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+ %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
%v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
%v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
%v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+ %nxv16i8 = call <vscale x 16 x i8> @llvm.pext.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i8> poison)
%nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
%nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
%nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
diff --git a/llvm/test/Analysis/CostModel/X86/pdep-pext.ll b/llvm/test/Analysis/CostModel/X86/pdep-pext.ll
new file mode 100644
index 0000000000000..ed26108505780
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/pdep-pext.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=x86_64-- | FileCheck %s --check-prefixes=NOBMI2
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=x86_64-- -mattr=+bmi2 | FileCheck %s --check-prefixes=BMI2
+
+; With +bmi2 pdep/pext are native (Legal for i32/i64, Promote for i8/i16).
+; Without bmi2 they expand to the Hacker's Delight scalar sequence.
+; There is no vector pdep/pext, so vectors always scalarize.
+
+define void @pdep() {
+; NOBMI2-LABEL: 'pdep'
+; NOBMI2-NEXT: Cost Model: Found costs of 95 for: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 222 for: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:252 CodeSize:252 Lat:492 SizeLat:252 for: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:398 CodeSize:302 Lat:782 SizeLat:398 for: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:5478 CodeSize:5463 Lat:4531 SizeLat:4531 for: %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 1551 for: %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 1784 for: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:1015 CodeSize:1015 Lat:1975 SizeLat:1015 for: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:799 CodeSize:607 Lat:1567 SizeLat:799 for: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; BMI2-LABEL: 'pdep'
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+; BMI2-NEXT: Cost Model: Found costs of RThru:5478 CodeSize:5463 Lat:4531 SizeLat:4531 for: %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+; BMI2-NEXT: Cost Model: Found costs of 47 for: %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BMI2-NEXT: Cost Model: Found costs of 16 for: %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BMI2-NEXT: Cost Model: Found costs of 11 for: %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BMI2-NEXT: Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
+ %i16 = call i16 @llvm.pdep.i16(i16 poison, i16 poison)
+ %i32 = call i32 @llvm.pdep.i32(i32 poison, i32 poison)
+ %i64 = call i64 @llvm.pdep.i64(i64 poison, i64 poison)
+ %i128 = call i128 @llvm.pdep.i128(i128 poison, i128 poison)
+ %v16i8 = call <16 x i8> @llvm.pdep.v16i8(<16 x i8> poison, <16 x i8> poison)
+ %v8i16 = call <8 x i16> @llvm.pdep.v8i16(<8 x i16> poison, <8 x i16> poison)
+ %v4i32 = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> poison, <4 x i32> poison)
+ %v2i64 = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> poison, <2 x i64> poison)
+ ret void
+}
+
+define void @pext() {
+; NOBMI2-LABEL: 'pext'
+; NOBMI2-NEXT: Cost Model: Found costs of 95 for: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 222 for: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:252 CodeSize:252 Lat:492 SizeLat:252 for: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:398 CodeSize:302 Lat:782 SizeLat:398 for: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:5478 CodeSize:5463 Lat:4531 SizeLat:4531 for: %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 1551 for: %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of 1784 for: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:1015 CodeSize:1015 Lat:1975 SizeLat:1015 for: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:799 CodeSize:607 Lat:1567 SizeLat:799 for: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; NOBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; BMI2-LABEL: 'pext'
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+; BMI2-NEXT: Cost Model: Found costs of 1 for: %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+; BMI2-NEXT: Cost Model: Found costs of RThru:5478 CodeSize:5463 Lat:4531 SizeLat:4531 for: %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+; BMI2-NEXT: Cost Model: Found costs of 47 for: %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BMI2-NEXT: Cost Model: Found costs of 16 for: %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BMI2-NEXT: Cost Model: Found costs of 11 for: %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BMI2-NEXT: Cost Model: Found costs of 5 for: %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
+ %i16 = call i16 @llvm.pext.i16(i16 poison, i16 poison)
+ %i32 = call i32 @llvm.pext.i32(i32 poison, i32 poison)
+ %i64 = call i64 @llvm.pext.i64(i64 poison, i64 poison)
+ %i128 = call i128 @llvm.pext.i128(i128 poison, i128 poison)
+ %v16i8 = call <16 x i8> @llvm.pext.v16i8(<16 x i8> poison, <16 x i8> poison)
+ %v8i16 = call <8 x i16> @llvm.pext.v8i16(<8 x i16> poison, <8 x i16> poison)
+ %v4i32 = call <4 x i32> @llvm.pext.v4i32(<4 x i32> poison, <4 x i32> poison)
+ %v2i64 = call <2 x i64> @llvm.pext.v2i64(<2 x i64> poison, <2 x i64> poison)
+ ret void
+}
>From be0937c669ff9d35a6eab67d958532907c112ad9 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Wed, 5 Aug 2026 23:41:36 -0700
Subject: [PATCH 5/5] Handle wider scalar and vscale types
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 5 +++--
.../Analysis/CostModel/AArch64/pdep-pext.ll | 18 ++++++++++++++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 10c38d7212405..5c32101193681 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2822,8 +2822,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
const TargetLoweringBase *TLI = getTLI();
- // pdep/pext are not lane-separable
- bool BitManipSplit = (ISD == ISD::PDEP || ISD == ISD::PEXT) && LT.first > 1;
+ // Scalar pdep/pext are not lane-separable
+ bool BitManipSplit = (ISD == ISD::PDEP || ISD == ISD::PEXT) &&
+ LT.first > 1 && !LT.second.isVector();
if (!BitManipSplit && TLI->isOperationLegalOrPromote(ISD, LT.second)) {
if (IID == Intrinsic::fabs && LT.second.isFloatingPoint() &&
diff --git a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
index 085a3d3a2c1ab..a7ad8781ac3d9 100644
--- a/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/pdep-pext.ll
@@ -21,6 +21,9 @@ define void @pdep() {
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv32i8 = call <vscale x 32 x i8> @llvm.pdep.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i32 = call <vscale x 8 x i32> @llvm.pdep.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i64 = call <vscale x 4 x i64> @llvm.pdep.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-BITPERM-LABEL: 'pdep'
@@ -37,6 +40,9 @@ define void @pdep() {
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv32i8 = call <vscale x 32 x i8> @llvm.pdep.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv8i32 = call <vscale x 8 x i32> @llvm.pdep.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.pdep.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%i8 = call i8 @llvm.pdep.i8(i8 poison, i8 poison)
@@ -52,6 +58,9 @@ define void @pdep() {
%nxv8i16 = call <vscale x 8 x i16> @llvm.pdep.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
%nxv4i32 = call <vscale x 4 x i32> @llvm.pdep.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
%nxv2i64 = call <vscale x 2 x i64> @llvm.pdep.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+ %nxv32i8 = call <vscale x 32 x i8> @llvm.pdep.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+ %nxv8i32 = call <vscale x 8 x i32> @llvm.pdep.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+ %nxv4i64 = call <vscale x 4 x i64> @llvm.pdep.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
ret void
}
@@ -70,6 +79,9 @@ define void @pext() {
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv32i8 = call <vscale x 32 x i8> @llvm.pext.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv8i32 = call <vscale x 8 x i32> @llvm.pext.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of Invalid for: %nxv4i64 = call <vscale x 4 x i64> @llvm.pext.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
; CHECK-NO-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-BITPERM-LABEL: 'pext'
@@ -86,6 +98,9 @@ define void @pext() {
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of 2 for: %nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv32i8 = call <vscale x 32 x i8> @llvm.pext.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv8i32 = call <vscale x 8 x i32> @llvm.pext.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+; CHECK-BITPERM-NEXT: Cost Model: Found costs of 4 for: %nxv4i64 = call <vscale x 4 x i64> @llvm.pext.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
; CHECK-BITPERM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%i8 = call i8 @llvm.pext.i8(i8 poison, i8 poison)
@@ -101,5 +116,8 @@ define void @pext() {
%nxv8i16 = call <vscale x 8 x i16> @llvm.pext.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i16> poison)
%nxv4i32 = call <vscale x 4 x i32> @llvm.pext.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
%nxv2i64 = call <vscale x 2 x i64> @llvm.pext.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i64> poison)
+ %nxv32i8 = call <vscale x 32 x i8> @llvm.pext.nxv32i8(<vscale x 32 x i8> poison, <vscale x 32 x i8> poison)
+ %nxv8i32 = call <vscale x 8 x i32> @llvm.pext.nxv8i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison)
+ %nxv4i64 = call <vscale x 4 x i64> @llvm.pext.nxv4i64(<vscale x 4 x i64> poison, <vscale x 4 x i64> poison)
ret void
}
More information about the llvm-commits
mailing list