[llvm] 921f4da - [SLP][Modularisation][NFC] Extract free cost helpers into SLPCostAnalysis (#210278)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 23:00:28 PDT 2026
Author: Madhur Amilkanthwar
Date: 2026-07-21T06:00:21Z
New Revision: 921f4da4a05f93391937ccc337aa448b3d8de539
URL: https://github.com/llvm/llvm-project/commit/921f4da4a05f93391937ccc337aa448b3d8de539
DIFF: https://github.com/llvm/llvm-project/commit/921f4da4a05f93391937ccc337aa448b3d8de539.diff
LOG: [SLP][Modularisation][NFC] Extract free cost helpers into SLPCostAnalysis (#210278)
Move the BoUpSLP-independent cost helpers out of SLPVectorizer.cpp into
SLPVectorizer/SLPCostAnalysis.{h,cpp} (namespace llvm::slpvectorizer).
Moved:
* getShuffleCost
* getGEPCosts
RFC: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
Added:
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
Modified:
llvm/lib/Transforms/Vectorize/CMakeLists.txt
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index cb132d580cfe8..4f46483b4341d 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -23,6 +23,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Scheduler.cpp
SandboxVectorizer/SeedCollector.cpp
SandboxVectorizer/VecUtils.cpp
+ SLPVectorizer/SLPCostAnalysis.cpp
SLPVectorizer/SLPUtils.cpp
SLPVectorizer.cpp
Vectorize.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9650415998142..46300da21e0a1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17,6 +17,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
+#include "SLPVectorizer/SLPCostAnalysis.h"
#include "SLPVectorizer/SLPUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
@@ -6676,40 +6677,6 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
return Stride;
}
-static std::pair<InstructionCost, InstructionCost>
-getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
- Value *BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind,
- Type *ScalarTy, VectorType *VecTy);
-
-/// Returns the cost of the shuffle instructions with the given \p Kind, vector
-/// type \p Tp and optional \p Mask. Adds SLP-specifc cost estimation for insert
-/// subvector pattern.
-static InstructionCost
-getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
- VectorType *Tp, ArrayRef<int> Mask = {},
- TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
- int Index = 0, VectorType *SubTp = nullptr,
- ArrayRef<const Value *> Args = {}) {
- VectorType *DstTy = Tp;
- if (!Mask.empty())
- DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size());
-
- if (Kind != TTI::SK_PermuteTwoSrc)
- return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
- int NumSrcElts = Tp->getElementCount().getKnownMinValue();
- int NumSubElts;
- if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
- Mask, NumSrcElts, NumSubElts, Index)) {
- if (Index + NumSubElts > NumSrcElts &&
- Index + NumSrcElts <= static_cast<int>(Mask.size()))
- return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, Mask,
- TTI::TCK_RecipThroughput, Index, Tp);
- }
- return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
-}
-
/// This is similar to TargetTransformInfo::getScalarizationOverhead, but if
/// ScalarTy is a FixedVectorType, a vector will be inserted or extracted
/// instead of a scalar.
@@ -6983,7 +6950,7 @@ static bool isMaskedLoadCompress(
// shuffle cost estimation in that case.
if (VectorGEPCost + LoadCost >= GatherCost)
return false;
- InstructionCost CompressCost = ::getShuffleCost(
+ InstructionCost CompressCost = getShuffleCost(
TTI, TTI::SK_PermuteSingleSrc, LoadVecTy, CompressMask, CostKind);
if (!Order.empty()) {
SmallVector<int> NewMask(Sz, PoisonMaskElem);
@@ -7553,7 +7520,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
getScalarizationOverhead(
TTI, PtrScalarTy, PtrVecTy, APInt::getOneBitSet(Sz, 0),
/*Insert=*/true, /*Extract=*/false, CostKind) +
- ::getShuffleCost(TTI, TTI::SK_Broadcast, PtrVecTy, {}, CostKind);
+ getShuffleCost(TTI, TTI::SK_Broadcast, PtrVecTy, {}, CostKind);
// The cost of scalar loads.
InstructionCost ScalarLoadsCost =
accumulate(VL, InstructionCost(),
@@ -7658,8 +7625,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
getScalarizationOverhead(
TTI, ScalarTy, SubVecTy, APInt::getOneBitSet(SliceVF, 0),
/*Insert=*/true, /*Extract=*/false, CostKind) +
- ::getShuffleCost(TTI, TTI::SK_Broadcast, SubVecTy, {},
- CostKind);
+ getShuffleCost(TTI, TTI::SK_Broadcast, SubVecTy, {}, CostKind);
}
switch (LS) {
case LoadsState::Vectorize:
@@ -7684,8 +7650,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
Intrinsic::masked_load, SubVecTy,
CommonAlignment, LI0->getPointerAddressSpace()),
CostKind) +
- ::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy,
- {}, CostKind);
+ getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy,
+ {}, CostKind);
break;
case LoadsState::ScatterVectorize:
VecLdCost += TTI.getMemIntrinsicInstrCost(
@@ -7705,8 +7671,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
ShuffleMask[Idx] = Idx / VF == SliceIdx ? VL.size() + Idx % VF : Idx;
if (SliceStart > 0)
VecLdCost +=
- ::getShuffleCost(TTI, TTI::SK_InsertSubvector, VecTy, ShuffleMask,
- CostKind, SliceStart, SubVecTy);
+ getShuffleCost(TTI, TTI::SK_InsertSubvector, VecTy, ShuffleMask,
+ CostKind, SliceStart, SubVecTy);
}
// If masked gather cost is higher - better to vectorize, so
// consider it as a gather node. It will be better estimated
@@ -8252,7 +8218,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
InstructionCost PermuteCost =
TopToBottom
? 0
- : ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, Ty, Mask);
+ : getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, Ty, Mask);
InstructionCost InsertFirstCost = TTI->getVectorInstrCost(
Instruction::InsertElement, Ty, TTI::TCK_RecipThroughput, 0,
PoisonValue::get(Ty), *It);
@@ -11331,7 +11297,7 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
return std::make_pair(true, false);
}
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- InstructionCost ReusesCost = ::getShuffleCost(
+ InstructionCost ReusesCost = getShuffleCost(
TTI, TTI::SK_PermuteSingleSrc, VecTy,
NumUniqueScalarValues > VL.size() / 2 ? ArrayRef<int>()
: ArrayRef(ReuseShuffleIndices),
@@ -11512,12 +11478,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
if (NumParts >= VL.size())
return false;
constexpr TTI::TargetCostKind Kind = TTI::TCK_RecipThroughput;
- InstructionCost InsertCost = ::getShuffleCost(
+ InstructionCost InsertCost = getShuffleCost(
*TTI, TTI::SK_InsertSubvector, VecTy, {}, Kind, Op1.size(), Op2VecTy);
auto *SubVecTy = cast<VectorType>(
getWidenedType(ScalarTy, std::max(Op1.size(), Op2.size())));
InstructionCost NewShuffleCost =
- ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, SubVecTy, Mask, Kind);
+ getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, SubVecTy, Mask, Kind);
if (!LocalState.isCmpOp() && NumParts <= 1 &&
(Mask.empty() || InsertCost >= NewShuffleCost))
return false;
@@ -11538,8 +11504,8 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
OriginalMask[Idx] = Idx + (Op1Indices.test(Idx) ? 0 : VL.size());
}
InstructionCost OriginalCost =
- OriginalVecOpsCost + ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
- VecTy, OriginalMask, Kind);
+ OriginalVecOpsCost +
+ getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, OriginalMask, Kind);
InstructionCost NewVecOpsCost =
TTI->getArithmeticInstrCost(Opcode0, Op1VecTy, Kind) +
TTI->getArithmeticInstrCost(Opcode1, Op2VecTy, Kind);
@@ -12645,7 +12611,7 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
Type *ScalarTy = VL.front()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
InstructionCost VectorizeCostEstimate =
- ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, {}, Kind) +
+ getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, {}, Kind) +
::getScalarizationOverhead(*TTI, ScalarTy, VecTy, Extracted,
/*Insert=*/false, /*Extract=*/true, Kind);
InstructionCost ScalarizeCostEstimate = ::getScalarizationOverhead(
@@ -14160,88 +14126,6 @@ class BaseShuffleAnalysis {
};
} // namespace
-/// Calculate the scalar and the vector costs from vectorizing set of GEPs.
-static std::pair<InstructionCost, InstructionCost>
-getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
- Value *BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind,
- Type *ScalarTy, VectorType *VecTy) {
- InstructionCost ScalarCost = 0;
- InstructionCost VecCost = 0;
- // Here we
diff erentiate two cases: (1) when Ptrs represent a regular
- // vectorization tree node (as they are pointer arguments of scattered
- // loads) or (2) when Ptrs are the arguments of loads or stores being
- // vectorized as plane wide unit-stride load/store since all the
- // loads/stores are known to be from/to adjacent locations.
- if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
- // Case 2: estimate costs for pointer related costs when vectorizing to
- // a wide load/store.
- // Scalar cost is estimated as a set of pointers with known relationship
- // between them.
- // For vector code we will use BasePtr as argument for the wide load/store
- // but we also need to account all the instructions which are going to
- // stay in vectorized code due to uses outside of these scalar
- // loads/stores.
- ScalarCost = TTI.getPointersChainCost(
- Ptrs, BasePtr, TTI::PointersChainInfo::getUnitStride(), ScalarTy,
- CostKind);
-
- SmallVector<const Value *> PtrsRetainedInVecCode;
- for (Value *V : Ptrs) {
- if (V == BasePtr) {
- PtrsRetainedInVecCode.push_back(V);
- continue;
- }
- auto *Ptr = dyn_cast<GetElementPtrInst>(V);
- // For simplicity assume Ptr to stay in vectorized code if it's not a
- // GEP instruction. We don't care since it's cost considered free.
- // TODO: We should check for any uses outside of vectorizable tree
- // rather than just single use.
- if (!Ptr || !Ptr->hasOneUse())
- PtrsRetainedInVecCode.push_back(V);
- }
-
- if (PtrsRetainedInVecCode.size() == Ptrs.size()) {
- // If all pointers stay in vectorized code then we don't have
- // any savings on that.
- return std::make_pair(TTI::TCC_Free, TTI::TCC_Free);
- }
- VecCost = TTI.getPointersChainCost(PtrsRetainedInVecCode, BasePtr,
- TTI::PointersChainInfo::getKnownStride(),
- VecTy, CostKind);
- } else {
- // Case 1: Ptrs are the arguments of loads that we are going to transform
- // into masked gather load intrinsic.
- // All the scalar GEPs will be removed as a result of vectorization.
- // For any external uses of some lanes extract element instructions will
- // be generated (which cost is estimated separately).
- TTI::PointersChainInfo PtrsInfo =
- all_of(Ptrs,
- [](const Value *V) {
- auto *Ptr = dyn_cast<GetElementPtrInst>(V);
- return Ptr && !Ptr->hasAllConstantIndices();
- })
- ? TTI::PointersChainInfo::getUnknownStride()
- : TTI::PointersChainInfo::getKnownStride();
-
- ScalarCost =
- TTI.getPointersChainCost(Ptrs, BasePtr, PtrsInfo, ScalarTy, CostKind);
- auto *BaseGEP = dyn_cast<GEPOperator>(BasePtr);
- if (!BaseGEP) {
- auto *It = find_if(Ptrs, IsaPred<GEPOperator>);
- if (It != Ptrs.end())
- BaseGEP = cast<GEPOperator>(*It);
- }
- if (BaseGEP) {
- SmallVector<const Value *> Indices(BaseGEP->indices());
- VecCost = TTI.getGEPCost(BaseGEP->getSourceElementType(),
- BaseGEP->getPointerOperand(), Indices, VecTy,
- CostKind);
- }
- }
-
- return std::make_pair(ScalarCost, VecCost);
-}
-
void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
assert(TE.isGather() && TE.ReorderIndices.empty() &&
"Expected gather node without reordering.");
@@ -14358,9 +14242,8 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
auto *ScalarTy = TE.Scalars.front()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, TE.Scalars.size()));
for (auto [Idx, Sz] : SubVectors) {
- Cost +=
- ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {}, CostKind,
- Idx, cast<VectorType>(getWidenedType(ScalarTy, Sz)));
+ Cost += getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {}, CostKind,
+ Idx, cast<VectorType>(getWidenedType(ScalarTy, Sz)));
}
Cost += getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
/*Insert=*/true,
@@ -14376,11 +14259,11 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
ReorderMask[I] = I + TE.ReorderIndices.size();
}
}
- Cost += ::getShuffleCost(*TTI,
- any_of(ReorderMask, [&](int I) { return I >= Sz; })
- ? TTI::SK_PermuteTwoSrc
- : TTI::SK_PermuteSingleSrc,
- VecTy, ReorderMask);
+ Cost += getShuffleCost(*TTI,
+ any_of(ReorderMask, [&](int I) { return I >= Sz; })
+ ? TTI::SK_PermuteTwoSrc
+ : TTI::SK_PermuteSingleSrc,
+ VecTy, ReorderMask);
DemandedElts = APInt::getAllOnes(TE.Scalars.size());
ReorderMask.assign(Sz, PoisonMaskElem);
for (unsigned I : seq<unsigned>(Sz)) {
@@ -14397,7 +14280,7 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
/*Insert=*/true, /*Extract=*/false, CostKind);
if (!DemandedElts.isAllOnes())
- BVCost += ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, ReorderMask);
+ BVCost += getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, ReorderMask);
if (Cost >= BVCost) {
SmallVector<int> Mask(TE.ReorderIndices.begin(), TE.ReorderIndices.end());
reorderScalars(TE.Scalars, Mask);
@@ -14593,8 +14476,8 @@ bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
fixupOrderingIndices(Order);
SmallVector<int> Mask;
inversePermutation(Order, Mask);
- BitcastCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, SrcVecTy,
- Mask, CostKind);
+ BitcastCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, SrcVecTy,
+ Mask, CostKind);
}
// Check if the combination can be modeled as a bitcast+byteswap operation.
constexpr unsigned ByteSize = 8;
@@ -15105,7 +14988,7 @@ void BoUpSLP::transformNodes() {
TTI->getMemoryOpCost(Instruction::Load, VecTy, BaseLI->getAlign(),
BaseLI->getPointerAddressSpace(), CostKind,
TTI::OperandValueInfo()) +
- ::getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
+ getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
VecTy, BaseLI->getPointerOperand(),
@@ -15146,7 +15029,7 @@ void BoUpSLP::transformNodes() {
TTI->getMemoryOpCost(Instruction::Store, VecTy, BaseSI->getAlign(),
BaseSI->getPointerAddressSpace(), CostKind,
TTI::OperandValueInfo()) +
- ::getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
+ getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, Mask, CostKind);
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
VecTy, BaseSI->getPointerOperand(),
@@ -15492,11 +15375,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
InstructionCost InsertCost =
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
PoisonValue::get(VecTy), *It);
- return InsertCost + ::getShuffleCost(TTI,
- TargetTransformInfo::SK_Broadcast,
- VecTy, ShuffleMask, CostKind,
- /*Index=*/0, /*SubTp=*/nullptr,
- /*Args=*/*It);
+ return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast,
+ VecTy, ShuffleMask, CostKind,
+ /*Index=*/0, /*SubTp=*/nullptr,
+ /*Args=*/*It);
}
return GatherCost +
(all_of(Gathers, IsaPred<UndefValue>)
@@ -15600,14 +15482,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
if (*ShuffleKinds[Part] != TTI::SK_PermuteSingleSrc ||
!ShuffleVectorInst::isIdentityMask(
MaskSlice, std::max<unsigned>(NumElts, MaskSlice.size())))
- Cost += ::getShuffleCost(
+ Cost += getShuffleCost(
TTI, *ShuffleKinds[Part],
cast<VectorType>(getWidenedType(ScalarTy, NumElts)), MaskSlice);
continue;
}
if (*RegShuffleKind != TTI::SK_PermuteSingleSrc ||
!ShuffleVectorInst::isIdentityMask(SubMask, EltsPerVector)) {
- Cost += ::getShuffleCost(
+ Cost += getShuffleCost(
TTI, *RegShuffleKind,
cast<VectorType>(getWidenedType(ScalarTy, EltsPerVector)), SubMask);
}
@@ -15616,7 +15498,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
for (const auto [Idx, SubVecSize] : zip(Indices, SubVecSizes)) {
assert((Idx + SubVecSize) <= BaseVF &&
"SK_ExtractSubvector index out of range");
- Cost += ::getShuffleCost(
+ Cost += getShuffleCost(
TTI, TTI::SK_ExtractSubvector,
cast<VectorType>(getWidenedType(ScalarTy, BaseVF)), {}, CostKind,
Idx, cast<VectorType>(getWidenedType(ScalarTy, SubVecSize)));
@@ -15625,7 +15507,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// subvector extract.
SubMask.assign(NumElts, PoisonMaskElem);
copy(MaskSlice, SubMask.begin());
- InstructionCost OriginalCost = ::getShuffleCost(
+ InstructionCost OriginalCost = getShuffleCost(
TTI, *ShuffleKinds[Part],
cast<VectorType>(getWidenedType(ScalarTy, NumElts)), SubMask);
if (OriginalCost < Cost)
@@ -15722,8 +15604,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return ::getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
- cast<VectorType>(V1->getType()), Mask);
+ return getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(V1->getType()), Mask);
}
InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask,
ArrayRef<Value *> VL) const {
@@ -15732,7 +15614,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return ::getShuffleCost(
+ return getShuffleCost(
TTI, TTI::SK_PermuteSingleSrc, cast<VectorType>(V1->getType()), Mask,
TTI::TCK_RecipThroughput, /*Index=*/0, /*SubTp=*/nullptr, VL);
}
@@ -16302,7 +16184,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
I1 = I2 + CommonMask.size();
}
}
- Cost += ::getShuffleCost(
+ Cost += getShuffleCost(
TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(getWidenedType(ScalarTy, CommonMask.size())),
SVMask, CostKind);
@@ -16326,7 +16208,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
getWidenedType(EScalarTy, E->getVectorFactor()),
TTI::CastContextHint::Normal, CostKind);
}
- Cost += ::getShuffleCost(
+ Cost += getShuffleCost(
TTI, TTI::SK_InsertSubvector,
cast<VectorType>(getWidenedType(ScalarTy, CommonMask.size())), {},
CostKind, Idx,
@@ -16761,7 +16643,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(E->ReuseShuffleIndices.empty() && "Expected empty reuses mask.");
InstructionCost VectorCost = 0;
if (E->ReorderIndices.empty()) {
- VectorCost = ::getShuffleCost(
+ VectorCost = getShuffleCost(
*TTI, TTI::SK_InsertSubvector, cast<VectorType>(FinalVecTy), {},
CostKind, E->CombinedEntriesWithIndices.back().second,
cast<VectorType>(getWidenedType(
@@ -16775,9 +16657,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
VectorizableTree[E->CombinedEntriesWithIndices.back().first]
->getVectorFactor());
VectorCost =
- ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
- cast<VectorType>(getWidenedType(ScalarTy, CommonVF)),
- E->getSplitMask(), CostKind);
+ getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(getWidenedType(ScalarTy, CommonVF)),
+ E->getSplitMask(), CostKind);
}
VectorCost += SpillsReloads;
LLVM_DEBUG(dumpTreeCosts(E, 0, VectorCost, 0, "Calculated costs for Tree"));
@@ -16803,9 +16685,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
assert(!isa<StructType>(FinalVecTy) &&
"Expected non-struct vector type for shuffle cost calculation.");
- CommonCost = ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- cast<VectorType>(FinalVecTy), Mask, CostKind,
- /*Index=*/0, cast<VectorType>(VecTy));
+ CommonCost = getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(FinalVecTy), Mask, CostKind,
+ /*Index=*/0, cast<VectorType>(VecTy));
}
assert((E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::ScatterVectorize ||
@@ -17115,8 +16997,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// we can merge this shuffle with the following SK_Select.
auto *InsertVecTy = cast<VectorType>(getWidenedType(ScalarTy, InsertVecSz));
if (!IsIdentity)
- Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
- InsertVecTy, Mask);
+ Cost += getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
+ InsertVecTy, Mask);
auto *FirstInsert = cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
}));
@@ -17134,8 +17016,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (!InMask.all() && NumScalars != NumElts && !IsWholeSubvector) {
if (InsertVecSz != VecSz) {
auto *ActualVecTy = cast<VectorType>(getWidenedType(ScalarTy, VecSz));
- Cost += ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, ActualVecTy, {},
- CostKind, OffsetBeg - Offset, InsertVecTy);
+ Cost += getShuffleCost(*TTI, TTI::SK_InsertSubvector, ActualVecTy, {},
+ CostKind, OffsetBeg - Offset, InsertVecTy);
} else {
for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
Mask[I] = InMask.test(I) ? PoisonMaskElem : I;
@@ -17146,8 +17028,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I)
Mask[I] =
((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I;
- Cost +=
- ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
+ Cost += getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
}
}
if (ShuffleOrOp == Instruction::InsertValue &&
@@ -17374,7 +17255,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (CondNumElements != VecTyNumElements) {
// When the return type is i1 but the source is fixed vector type,
// we need to duplicate the condition value.
- VecCost += ::getShuffleCost(
+ VecCost += getShuffleCost(
*TTI, TTI::SK_PermuteSingleSrc, MaskTy,
createReplicatedMask(VecTyNumElements / CondNumElements,
CondNumElements));
@@ -17675,15 +17556,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
LI0->getPointerAddressSpace()),
CostKind);
// TODO: include this cost into CommonCost.
- VecLdCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- LoadVecTy, CompressMask, CostKind);
+ VecLdCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, LoadVecTy,
+ CompressMask, CostKind);
} else {
VecLdCost = TTI->getMemoryOpCost(
Instruction::Load, LoadVecTy, CommonAlignment,
LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
// TODO: include this cost into CommonCost.
- VecLdCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- LoadVecTy, CompressMask, CostKind);
+ VecLdCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, LoadVecTy,
+ CompressMask, CostKind);
}
break;
}
@@ -17922,8 +17803,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
*TLI);
},
Mask);
- VecCost += ::getShuffleCost(TTIRef, TargetTransformInfo::SK_PermuteTwoSrc,
- cast<VectorType>(FinalVecTy), Mask, CostKind);
+ VecCost += getShuffleCost(TTIRef, TargetTransformInfo::SK_PermuteTwoSrc,
+ cast<VectorType>(FinalVecTy), Mask, CostKind);
// Patterns like [fadd,fsub] can be combined into a single instruction
// in x86. Reordering them into [fsub,fadd] blocks this pattern. So we
// need to take into account their order when looking for the most used
@@ -17972,10 +17853,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
NextIndex += SV->getShuffleMask().size();
return true;
}))
- return ::getShuffleCost(
- *TTI, TargetTransformInfo::SK_PermuteSingleSrc,
- cast<VectorType>(VecTy),
- calculateShufflevectorMask(E->Scalars));
+ return getShuffleCost(*TTI,
+ TargetTransformInfo::SK_PermuteSingleSrc,
+ cast<VectorType>(VecTy),
+ calculateShufflevectorMask(E->Scalars));
}
return TTI::TCC_Free;
});
@@ -19322,8 +19203,8 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
if (!TE->ReuseShuffleIndices.empty())
addMask(Mask, TE->ReuseShuffleIndices);
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, EntryVF))
- GatherCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- cast<VectorType>(VecTy), Mask);
+ GatherCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(VecTy), Mask);
// If all scalars are reused in gather node(s) or other vector nodes, there
// might be extra cost for inserting them.
if ((!TE->hasState() || !TE->isAltShuffle()) &&
@@ -20154,7 +20035,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
- C = ::getShuffleCost(
+ C = getShuffleCost(
*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(getWidenedType(TE->getMainOp()->getType(), VecVF)),
OrigMask);
@@ -20173,10 +20054,10 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ResizeMask[Mask[I]] = Mask[I];
}
if (!ShuffleVectorInst::isIdentityMask(ResizeMask, VF))
- C = ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- cast<VectorType>(getWidenedType(
- TE->getMainOp()->getType(), VecVF)),
- ResizeMask);
+ C = getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(getWidenedType(
+ TE->getMainOp()->getType(), VecVF)),
+ ResizeMask);
LLVM_DEBUG(
dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of insertelement external users.\n";
@@ -20206,8 +20087,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
(Data.index() < VF &&
static_cast<int>(Data.index()) == Data.value());
})) {
- InstructionCost C = ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- cast<VectorType>(FTy), Mask);
+ InstructionCost C = getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(FTy), Mask);
C = ScaleCost(C, *TEs.front());
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of insertelement "
@@ -20225,8 +20106,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
VF = Mask.size();
}
auto *FTy = getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
- InstructionCost C = ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
- cast<VectorType>(FTy), Mask);
+ InstructionCost C = getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(FTy), Mask);
C = ScaleCost(C, *TEs.back());
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of vector node and external "
@@ -21132,10 +21013,10 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
ShuffleVectorInst::isDeInterleaveMaskOfFactor(
Mask, Entries.front()->getInterleaveFactor()))
return TTI::TCC_Free;
- return ::getShuffleCost(TTI,
- Entries.size() > 1 ? TTI::SK_PermuteTwoSrc
- : TTI::SK_PermuteSingleSrc,
- VecTy, Mask, CostKind);
+ return getShuffleCost(TTI,
+ Entries.size() > 1 ? TTI::SK_PermuteTwoSrc
+ : TTI::SK_PermuteSingleSrc,
+ VecTy, Mask, CostKind);
};
InstructionCost ShuffleCost = GetShuffleCost(SubMask, Entries, VecTy);
InstructionCost FirstShuffleCost = 0;
@@ -21331,8 +21212,8 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
any_of(VL, [](Value *V) { return !isa<UndefValue>(V) && isConstant(V); });
// 1. Shuffle input source vector and constant vector.
if (!ForPoisonSrc && IsAnyNonUndefConst) {
- Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteTwoSrc,
- cast<VectorType>(VecTy), ConstantShuffleMask);
+ Cost += getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteTwoSrc,
+ cast<VectorType>(VecTy), ConstantShuffleMask);
}
// 2. Insert unique non-constants.
@@ -22865,8 +22746,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
if (I != PoisonMaskElem)
NewMask[Idx] = Mask.size();
SplatCost +=
- ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
- cast<VectorType>(VecTy), NewMask, CostKind);
+ getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(VecTy), NewMask, CostKind);
InstructionCost BVCost = TTI->getVectorInstrCost(
Instruction::InsertElement, VecTy, CostKind,
*find_if(Mask, not_equal_to(PoisonMaskElem)), Vec, V);
@@ -22877,9 +22758,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
for (auto [Idx, I] : enumerate(BVMask))
if (I != PoisonMaskElem)
NewMask[Idx] = I;
- BVCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
- cast<VectorType>(VecTy), NewMask,
- CostKind);
+ BVCost +=
+ getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(VecTy), NewMask, CostKind);
}
return SplatCost <= BVCost;
};
@@ -31285,8 +31166,8 @@ class HorizontalReduction {
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
VectorCost +=
- ::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
- CostKind, I * ScalarTyNumElements, VecTy);
+ getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
+ CostKind, I * ScalarTyNumElements, VecTy);
}
// We get one less arithmetic instruction compared to number of
// reduced values. We are also passing CtxI so that the backend can
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
new file mode 100644
index 0000000000000..bfe1e69261c39
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
@@ -0,0 +1,131 @@
+//===- SLPCostAnalysis.cpp - SLP Vectorizer free cost helpers -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SLPCostAnalysis.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/Casting.h"
+
+#include <utility>
+
+using namespace llvm;
+
+namespace llvm::slpvectorizer {
+
+InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
+ TTI::ShuffleKind Kind, VectorType *Tp,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind,
+ int Index, VectorType *SubTp,
+ ArrayRef<const Value *> Args) {
+ VectorType *DstTy = Tp;
+ if (!Mask.empty())
+ DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size());
+
+ if (Kind != TTI::SK_PermuteTwoSrc)
+ return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
+ Args);
+ int NumSrcElts = Tp->getElementCount().getKnownMinValue();
+ int NumSubElts;
+ if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
+ Mask, NumSrcElts, NumSubElts, Index)) {
+ if (Index + NumSubElts > NumSrcElts &&
+ Index + NumSrcElts <= static_cast<int>(Mask.size()))
+ return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, Mask,
+ TTI::TCK_RecipThroughput, Index, Tp);
+ }
+ return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
+ Args);
+}
+
+std::pair<InstructionCost, InstructionCost>
+getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
+ Value *BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind,
+ Type *ScalarTy, VectorType *VecTy) {
+ InstructionCost ScalarCost = 0;
+ InstructionCost VecCost = 0;
+ // Here we
diff erentiate two cases: (1) when Ptrs represent a regular
+ // vectorization tree node (as they are pointer arguments of scattered
+ // loads) or (2) when Ptrs are the arguments of loads or stores being
+ // vectorized as plane wide unit-stride load/store since all the
+ // loads/stores are known to be from/to adjacent locations.
+ if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
+ // Case 2: estimate costs for pointer related costs when vectorizing to
+ // a wide load/store.
+ // Scalar cost is estimated as a set of pointers with known relationship
+ // between them.
+ // For vector code we will use BasePtr as argument for the wide load/store
+ // but we also need to account all the instructions which are going to
+ // stay in vectorized code due to uses outside of these scalar
+ // loads/stores.
+ ScalarCost = TTI.getPointersChainCost(
+ Ptrs, BasePtr, TTI::PointersChainInfo::getUnitStride(), ScalarTy,
+ CostKind);
+
+ SmallVector<const Value *> PtrsRetainedInVecCode;
+ for (Value *V : Ptrs) {
+ if (V == BasePtr) {
+ PtrsRetainedInVecCode.push_back(V);
+ continue;
+ }
+ auto *Ptr = dyn_cast<GetElementPtrInst>(V);
+ // For simplicity assume Ptr to stay in vectorized code if it's not a
+ // GEP instruction. We don't care since it's cost considered free.
+ // TODO: We should check for any uses outside of vectorizable tree
+ // rather than just single use.
+ if (!Ptr || !Ptr->hasOneUse())
+ PtrsRetainedInVecCode.push_back(V);
+ }
+
+ if (PtrsRetainedInVecCode.size() == Ptrs.size()) {
+ // If all pointers stay in vectorized code then we don't have
+ // any savings on that.
+ return std::make_pair(TTI::TCC_Free, TTI::TCC_Free);
+ }
+ VecCost = TTI.getPointersChainCost(PtrsRetainedInVecCode, BasePtr,
+ TTI::PointersChainInfo::getKnownStride(),
+ VecTy, CostKind);
+ } else {
+ // Case 1: Ptrs are the arguments of loads that we are going to transform
+ // into masked gather load intrinsic.
+ // All the scalar GEPs will be removed as a result of vectorization.
+ // For any external uses of some lanes extract element instructions will
+ // be generated (which cost is estimated separately).
+ TTI::PointersChainInfo PtrsInfo =
+ all_of(Ptrs,
+ [](const Value *V) {
+ auto *Ptr = dyn_cast<GetElementPtrInst>(V);
+ return Ptr && !Ptr->hasAllConstantIndices();
+ })
+ ? TTI::PointersChainInfo::getUnknownStride()
+ : TTI::PointersChainInfo::getKnownStride();
+
+ ScalarCost =
+ TTI.getPointersChainCost(Ptrs, BasePtr, PtrsInfo, ScalarTy, CostKind);
+ auto *BaseGEP = dyn_cast<GEPOperator>(BasePtr);
+ if (!BaseGEP) {
+ auto *It = find_if(Ptrs, IsaPred<GEPOperator>);
+ if (It != Ptrs.end())
+ BaseGEP = cast<GEPOperator>(*It);
+ }
+ if (BaseGEP) {
+ SmallVector<const Value *> Indices(BaseGEP->indices());
+ VecCost = TTI.getGEPCost(BaseGEP->getSourceElementType(),
+ BaseGEP->getPointerOperand(), Indices, VecTy,
+ CostKind);
+ }
+ }
+
+ return std::make_pair(ScalarCost, VecCost);
+}
+
+} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
new file mode 100644
index 0000000000000..a3ae27e9905c7
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
@@ -0,0 +1,53 @@
+//===- SLPCostAnalysis.h - SLP Vectorizer free cost helpers ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Internal header used by SLPVectorizer.cpp. It declares free cost helpers
+// that do not depend on BoUpSLP or any other SLP-private type. The bulk of
+// the SLP cost model still lives in SLPVectorizer.cpp because it references
+// BoUpSLP internals.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOSTANALYSIS_H
+#define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOSTANALYSIS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Support/InstructionCost.h"
+
+#include <utility>
+
+namespace llvm {
+class Type;
+class Value;
+class VectorType;
+} // namespace llvm
+
+namespace llvm::slpvectorizer {
+
+/// Returns the cost of the shuffle instructions with the given \p Kind, vector
+/// type \p Tp and optional \p Mask. Adds SLP-specific cost estimation for
+/// insert subvector pattern.
+InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
+ TargetTransformInfo::ShuffleKind Kind,
+ VectorType *Tp, ArrayRef<int> Mask = {},
+ TargetTransformInfo::TargetCostKind CostKind =
+ TargetTransformInfo::TCK_RecipThroughput,
+ int Index = 0, VectorType *SubTp = nullptr,
+ ArrayRef<const Value *> Args = {});
+
+/// Calculate the scalar and the vector costs from vectorizing set of GEPs.
+std::pair<InstructionCost, InstructionCost>
+getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
+ Value *BasePtr, unsigned Opcode,
+ TargetTransformInfo::TargetCostKind CostKind, Type *ScalarTy,
+ VectorType *VecTy);
+
+} // namespace llvm::slpvectorizer
+
+#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOSTANALYSIS_H
More information about the llvm-commits
mailing list