[llvm] d53e245 - [COST][NFC]Introduce OperandValueKind in getMemoryOpCost, NFC.
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 19 07:34:06 PDT 2022
Author: Alexey Bataev
Date: 2022-08-19T07:33:00-07:00
New Revision: d53e245951f861f1874f493591938779b7498cc8
URL: https://github.com/llvm/llvm-project/commit/d53e245951f861f1874f493591938779b7498cc8
DIFF: https://github.com/llvm/llvm-project/commit/d53e245951f861f1874f493591938779b7498cc8.diff
LOG: [COST][NFC]Introduce OperandValueKind in getMemoryOpCost, NFC.
Added OperandValueKind OpdInfo parameter to getMemoryOpCost functions to
better estimate cost with immediate values.
Part of D126885.
Added:
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/include/llvm/CodeGen/BasicTTIImpl.h
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index c579f77422893..ab9b701808202 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1173,6 +1173,7 @@ class TargetTransformInfo {
getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+ OperandValueKind OpdInfo = OK_AnyValue,
const Instruction *I = nullptr) const;
/// \return The cost of VP Load and Store instructions.
@@ -1740,11 +1741,10 @@ class TargetTransformInfo::Concept {
const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) = 0;
- virtual InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- Align Alignment,
- unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I) = 0;
+ virtual InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ OperandValueKind OpdInfo, const Instruction *I) = 0;
virtual InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment,
unsigned AddressSpace,
@@ -2307,9 +2307,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ OperandValueKind OpdInfo,
const Instruction *I) override {
- return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind, I);
+ return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
+ OpdInfo, I);
}
InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 620c2e4e93cf7..a11e540cb7d9b 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -595,6 +595,7 @@ class TargetTransformInfoImplBase {
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) const {
return 1;
}
@@ -1099,9 +1100,11 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
case Instruction::Store: {
auto *SI = cast<StoreInst>(U);
Type *ValTy = U->getOperand(0)->getType();
+ TTI::OperandValueProperties OpVP = TTI::OP_None;
+ TTI::OperandValueKind OpVK = TTI::getOperandInfo(U->getOperand(0), OpVP);
return TargetTTI->getMemoryOpCost(Opcode, ValTy, SI->getAlign(),
- SI->getPointerAddressSpace(),
- CostKind, I);
+ SI->getPointerAddressSpace(), CostKind,
+ OpVK, I);
}
case Instruction::Load: {
// FIXME: Arbitary cost which could come from the backend.
@@ -1122,8 +1125,8 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
LoadType = TI->getDestTy();
}
return TargetTTI->getMemoryOpCost(Opcode, LoadType, LI->getAlign(),
- LI->getPointerAddressSpace(),
- CostKind, I);
+ LI->getPointerAddressSpace(), CostKind,
+ TTI::OK_AnyValue, I);
}
case Instruction::Select: {
const Value *Op0, *Op1;
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 826fd9106f99a..f97c7f6a0aaad 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1221,10 +1221,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) {
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr) {
assert(!Src->isVoidTy() && "Invalid type");
// Assume types, such as structs, are expensive.
if (getTLI()->getValueType(DL, Src, true) == MVT::Other)
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 001075d07bf85..b0f8fdeead00f 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -899,11 +899,12 @@ InstructionCost TargetTransformInfo::getReplicationShuffleCost(
InstructionCost TargetTransformInfo::getMemoryOpCost(
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind, const Instruction *I) const {
+ TTI::TargetCostKind CostKind, TTI::OperandValueKind OpdInfo,
+ const Instruction *I) const {
assert((I == nullptr || I->getOpcode() == Opcode) &&
"Opcode should reflect passed instruction.");
- InstructionCost Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment,
- AddressSpace, CostKind, I);
+ InstructionCost Cost = TTIImpl->getMemoryOpCost(
+ Opcode, Src, Alignment, AddressSpace, CostKind, OpdInfo, I);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 06cc1bfb628ff..be12a2de8ef23 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2249,7 +2249,8 @@ InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
ElementCount LegalVF = LT.second.getVectorElementCount();
InstructionCost MemOpCost =
- getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I);
+ getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind,
+ TTI::OK_AnyValue, I);
// Add on an overhead cost for using gathers/scatters.
// TODO: At the moment this is applied unilaterally for all CPUs, but at some
// point we may want a per-CPU overhead.
@@ -2265,6 +2266,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
EVT VT = TLI->getValueType(DL, Ty, true);
// Type legalization can't handle structs
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 67094287ed389..94b09e2ee0708 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -208,10 +208,11 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
bool IsZeroCmp) const;
bool useNeonVector(const Type *Ty) const;
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index afafad702f321..9c7ed9c7ead9f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1450,6 +1450,7 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
@@ -1489,7 +1490,7 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
? ST->getMVEVectorCostFactor(CostKind)
: 1;
return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind, I);
+ CostKind, OpdInfo, I);
}
InstructionCost
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 72d28acf6d4a1..e4344f8486750 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -253,10 +253,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
const Instruction *CxtI = nullptr);
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment, unsigned AddressSpace,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 13b96642a1ae9..fadaa935320ee 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -161,6 +161,7 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
assert(Opcode == Instruction::Load || Opcode == Instruction::Store);
// TODO: Handle other cost kinds.
@@ -169,7 +170,7 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
if (Opcode == Instruction::Store)
return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind, I);
+ CostKind, OpdInfo, I);
if (Src->isVectorTy()) {
VectorType *VecTy = cast<VectorType>(Src);
@@ -209,8 +210,8 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
return (3 - LogA) * Cost * NumLoads;
}
- return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind, I);
+ return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
+ OpdInfo, I);
}
InstructionCost
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index e79e9d02e177e..b34c947417068 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -116,10 +116,11 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::TargetCostKind CostKind);
InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
const SCEV *S);
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind);
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 86c39cdf7f533..d8660ae844ce8 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1145,6 +1145,7 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Src, nullptr);
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 6058ff9158c4a..36e6fec2a36d2 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -127,10 +127,11 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 602bf3a670094..63e28486cf4e1 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -243,8 +243,9 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
auto &VTy = *cast<VectorType>(DataTy);
- InstructionCost MemOpCost = getMemoryOpCost(Opcode, VTy.getElementType(),
- Alignment, 0, CostKind, I);
+ InstructionCost MemOpCost =
+ getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind,
+ TTI::OK_AnyValue, I);
unsigned NumLoads = getEstimatedVLFor(&VTy);
return NumLoads * MemOpCost;
}
@@ -433,6 +434,16 @@ InstructionCost RISCVTTIImpl::getExtendedReductionCost(
getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
}
+InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+ MaybeAlign Alignment,
+ unsigned AddressSpace,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
+ const Instruction *I) {
+ return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
+ OpdInfo, I);
+}
+
void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 0a9808220e7df..b9b07f1c7b85b 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -125,6 +125,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
Optional<FastMathFlags> FMF,
TTI::TargetCostKind CostKind);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
+
bool isElementTypeLegalForScalableVector(Type *Ty) const {
return TLI->isLegalElementTypeForRVV(Ty);
}
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 8b079e4e70018..101dd3f80a334 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -1109,6 +1109,7 @@ InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
assert(!Src->isVoidTy() && "Invalid type");
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 0804d0330de49..80d777aaf2fff 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -111,10 +111,11 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue);
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 7f3108ed9da46..62cb370024cfa 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -4050,6 +4050,7 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
MaybeAlign Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo,
const Instruction *I) {
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput) {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index f5f6097111e9a..f0aefed2713c7 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -156,10 +156,11 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
int VF,
const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind);
- InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
- MaybeAlign Alignment, unsigned AddressSpace,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost
+ getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment,
+ unsigned AddressSpace, TTI::TargetCostKind CostKind,
+ TTI::OperandValueKind OpdInfo = TTI::OK_AnyValue,
+ const Instruction *I = nullptr);
InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e09e66b8ea8ab..09139a8920f36 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6394,12 +6394,15 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
"Stride should be 1 or -1 for consecutive memory access");
const Align Alignment = getLoadStoreAlignment(I);
InstructionCost Cost = 0;
- if (Legal->isMaskRequired(I))
+ if (Legal->isMaskRequired(I)) {
Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
CostKind);
- else
+ } else {
+ TTI::OperandValueProperties OpVP = TTI::OP_None;
+ TTI::OperandValueKind OpVK = TTI::getOperandInfo(I->getOperand(0), OpVP);
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
- CostKind, I);
+ CostKind, OpVK, I);
+ }
bool Reverse = ConsecutiveStride < 0;
if (Reverse)
@@ -6676,9 +6679,11 @@ LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
const Align Alignment = getLoadStoreAlignment(I);
unsigned AS = getLoadStoreAddressSpace(I);
+ TTI::OperandValueProperties OpVP = TTI::OP_None;
+ TTI::OperandValueKind OpVK = TTI::getOperandInfo(I->getOperand(0), OpVP);
return TTI.getAddressComputationCost(ValTy) +
TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS,
- TTI::TCK_RecipThroughput, I);
+ TTI::TCK_RecipThroughput, OpVK, I);
}
return getWideningCost(I, VF);
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 926c30bf8b212..0d10e2f7a7373 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6048,15 +6048,15 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
auto *LI = cast<LoadInst>(V);
ScalarsCost += TTI->getMemoryOpCost(
Instruction::Load, LI->getType(), LI->getAlign(),
- LI->getPointerAddressSpace(), CostKind, LI);
+ LI->getPointerAddressSpace(), CostKind, TTI::OK_AnyValue, LI);
}
auto *LI = cast<LoadInst>(E->getMainOp());
auto *LoadTy = FixedVectorType::get(LI->getType(), VF);
Align Alignment = LI->getAlign();
- GatherCost +=
- VectorizedCnt *
- TTI->getMemoryOpCost(Instruction::Load, LoadTy, Alignment,
- LI->getPointerAddressSpace(), CostKind, LI);
+ GatherCost += VectorizedCnt *
+ TTI->getMemoryOpCost(Instruction::Load, LoadTy, Alignment,
+ LI->getPointerAddressSpace(),
+ CostKind, TTI::OK_AnyValue, LI);
GatherCost += ScatterVectorizeCnt *
TTI->getGatherScatterOpCost(
Instruction::Load, LoadTy, LI->getPointerOperand(),
@@ -6447,8 +6447,9 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
case Instruction::Load: {
// Cost of wide load - cost of scalar loads.
Align Alignment = cast<LoadInst>(VL0)->getAlign();
- InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
- Instruction::Load, ScalarTy, Alignment, 0, CostKind, VL0);
+ InstructionCost ScalarEltCost =
+ TTI->getMemoryOpCost(Instruction::Load, ScalarTy, Alignment, 0,
+ CostKind, TTI::OK_AnyValue, VL0);
if (NeedToShuffleReuses) {
CommonCost -= (EntryVF - VL.size()) * ScalarEltCost;
}
@@ -6456,7 +6457,7 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
InstructionCost VecLdCost;
if (E->State == TreeEntry::Vectorize) {
VecLdCost = TTI->getMemoryOpCost(Instruction::Load, VecTy, Alignment, 0,
- CostKind, VL0);
+ CostKind, TTI::OK_AnyValue, VL0);
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unknown EntryState");
Align CommonAlignment = Alignment;
@@ -6476,11 +6477,23 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
auto *SI =
cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
Align Alignment = SI->getAlign();
+ TTI::OperandValueProperties OpVP = TTI::OP_None;
+ TTI::OperandValueKind OpVK = TTI::getOperandInfo(SI->getOperand(0), OpVP);
InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
- Instruction::Store, ScalarTy, Alignment, 0, CostKind, VL0);
+ Instruction::Store, ScalarTy, Alignment, 0, CostKind, OpVK, VL0);
InstructionCost ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
+ OpVK = TTI::OK_AnyValue;
+ if (all_of(E->Scalars,
+ [](Value *V) {
+ return isConstant(cast<Instruction>(V)->getOperand(0));
+ }) &&
+ any_of(E->Scalars, [](Value *V) {
+ Value *Op = cast<Instruction>(V)->getOperand(0);
+ return !isa<UndefValue>(Op) && !cast<Constant>(Op)->isZeroValue();
+ }))
+ OpVK = TTI::OK_NonUniformConstantValue;
InstructionCost VecStCost = TTI->getMemoryOpCost(
- Instruction::Store, VecTy, Alignment, 0, CostKind, VL0);
+ Instruction::Store, VecTy, Alignment, 0, CostKind, OpVK, VL0);
LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecStCost, ScalarStCost));
return CommonCost + VecStCost - ScalarStCost;
}
More information about the llvm-commits
mailing list