[llvm] [TTI][RISCV] Model cost of loading constants arms of selects and compares (PR #109824)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 24 16:00:10 PDT 2024
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/109824
>From 76fbd409496674104acb95452825545477ad07a2 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Mon, 23 Sep 2024 15:40:46 -0700
Subject: [PATCH 1/3] [TTI][RISCV] Model cost of loading constants arms of
selects and compares
This follows in the spirit of 7d82c99403f615f6236334e698720bf979959704, and extends the costing API for compares and selects to provide information about the operands passed in an analogous manner. This allows us to model the cost of materializing the vector constant, as some select-of-constants are significantly more expensive than others when you account for the cost of materializing the constants involved.
Fixes https://github.com/llvm/llvm-project/issues/109466
---
.../llvm/Analysis/TargetTransformInfo.h | 21 +++--
.../llvm/Analysis/TargetTransformInfoImpl.h | 10 ++-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 17 ++--
llvm/lib/Analysis/TargetTransformInfo.cpp | 7 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 14 ++--
.../AArch64/AArch64TargetTransformInfo.h | 10 ++-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 15 ++--
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 10 ++-
llvm/lib/Target/BPF/BPFTargetTransformInfo.h | 12 +--
.../Hexagon/HexagonTargetTransformInfo.cpp | 12 +--
.../Hexagon/HexagonTargetTransformInfo.h | 10 ++-
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 13 ++-
.../Target/PowerPC/PPCTargetTransformInfo.h | 10 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 79 ++++++++++++-------
.../Target/RISCV/RISCVTargetTransformInfo.h | 10 ++-
.../SystemZ/SystemZTargetTransformInfo.cpp | 15 ++--
.../SystemZ/SystemZTargetTransformInfo.h | 10 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 20 ++---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 10 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 ++++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +-
.../Analysis/CostModel/RISCV/rvv-select.ll | 8 +-
.../RISCV/select-profitability.ll | 18 +++--
24 files changed, 219 insertions(+), 145 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index cd69a8a371b6e8..89a85bc8a90864 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1371,11 +1371,15 @@ class TargetTransformInfo {
/// is an existing instruction that holds Opcode, it may be passed in the
/// 'I' parameter. The \p VecPred parameter can be used to indicate the select
/// is using a compare with the specified predicate as condition. When vector
- /// types are passed, \p VecPred must be used for all lanes.
+ /// types are passed, \p VecPred must be used for all lanes. For a
+ /// comparison, the two operands are the natural values. For a select, the
+ /// two operands are the *value* operands, not the condition operand.
InstructionCost
getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+ OperandValueInfo Op1Info = {OK_AnyValue, OP_None},
+ OperandValueInfo Op2Info = {OK_AnyValue, OP_None},
const Instruction *I = nullptr) const;
/// \return The expected cost of vector Insert and Extract.
@@ -2049,11 +2053,11 @@ class TargetTransformInfo::Concept {
virtual InstructionCost getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) = 0;
- virtual InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) = 0;
+ virtual InstructionCost
+ getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
+ CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind,
+ OperandValueInfo Op1Info, OperandValueInfo Op2Info,
+ const Instruction *I) = 0;
virtual InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
unsigned Index, Value *Op0,
@@ -2710,8 +2714,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
+ OperandValueInfo Op1Info,
+ OperandValueInfo Op2Info,
const Instruction *I) override {
- return Impl.getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return Impl.getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
}
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 79c8bafbc6c0df..eca8818cc25e62 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -666,6 +666,8 @@ class TargetTransformInfoImplBase {
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info,
const Instruction *I) const {
return 1;
}
@@ -1332,19 +1334,23 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
match(U, m_LogicalOr()) ? Instruction::Or : Instruction::And, Ty,
CostKind, Op1Info, Op2Info, Operands, I);
}
+ const auto Op1Info = TTI::getOperandInfo(Operands[1]);
+ const auto Op2Info = TTI::getOperandInfo(Operands[2]);
Type *CondTy = Operands[0]->getType();
return TargetTTI->getCmpSelInstrCost(Opcode, U->getType(), CondTy,
CmpInst::BAD_ICMP_PREDICATE,
- CostKind, I);
+ CostKind, Op1Info, Op2Info, I);
}
case Instruction::ICmp:
case Instruction::FCmp: {
+ const auto Op1Info = TTI::getOperandInfo(Operands[0]);
+ const auto Op2Info = TTI::getOperandInfo(Operands[1]);
Type *ValTy = Operands[0]->getType();
// TODO: Also handle ICmp/FCmp constant expressions.
return TargetTTI->getCmpSelInstrCost(Opcode, ValTy, U->getType(),
I ? cast<CmpInst>(I)->getPredicate()
: CmpInst::BAD_ICMP_PREDICATE,
- CostKind, I);
+ CostKind, Op1Info, Op2Info, I);
}
case Instruction::InsertElement: {
auto *IE = dyn_cast<InsertElementInst>(U);
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 7198e134a2d262..ed074ecaebcf53 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1222,10 +1222,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return BaseT::getCFInstrCost(Opcode, CostKind, I);
}
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) {
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr) {
const TargetLoweringBase *TLI = getTLI();
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
@@ -1233,7 +1235,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
// Selects on vectors are actually vector selects.
if (ISD == ISD::SELECT) {
@@ -1260,8 +1262,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements();
if (CondTy)
CondTy = CondTy->getScalarType();
- InstructionCost Cost = thisT()->getCmpSelInstrCost(
- Opcode, ValVTy->getScalarType(), CondTy, VecPred, CostKind, I);
+ InstructionCost Cost =
+ thisT()->getCmpSelInstrCost(Opcode, ValVTy->getScalarType(), CondTy,
+ VecPred, CostKind, Op1Info, Op2Info, I);
// Return the cost of multiple scalar invocation plus the cost of
// inserting and extracting the values.
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 67b626f300a101..b5195f764cbd1c 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1015,11 +1015,12 @@ InstructionCost TargetTransformInfo::getCFInstrCost(
InstructionCost TargetTransformInfo::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind, const Instruction *I) const {
+ TTI::TargetCostKind CostKind, OperandValueInfo Op1Info,
+ OperandValueInfo Op2Info, const Instruction *I) const {
assert((I == nullptr || I->getOpcode() == Opcode) &&
"Opcode should reflect passed instruction.");
- InstructionCost Cost =
- TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ InstructionCost Cost = TTIImpl->getCmpSelInstrCost(
+ Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ac05a44abc2dd9..7a07bb67e77de9 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3440,15 +3440,14 @@ InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty,
return 1;
}
-InstructionCost AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
int ISD = TLI->InstructionOpcodeToISD(Opcode);
// We don't lower some vector selects well that are wider than the register
@@ -3527,7 +3526,8 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// The base case handles scalable vectors fine for now, since it treats the
// cost as 1 * legalization cost.
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
}
AArch64TTIImpl::TTI::MemCmpExpansionOptions
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 22bba21eedcc5a..28e45207596ecd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -208,10 +208,12 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
const SCEV *Ptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
bool IsZeroCmp) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 9b5349241c341b..865e2f3066ef01 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -934,11 +934,10 @@ InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);
}
-InstructionCost ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost ARMTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
// Thumb scalar code size cost for select.
@@ -1052,7 +1051,7 @@ InstructionCost ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
VecValTy->getNumElements() *
getCmpSelInstrCost(Opcode, ValTy->getScalarType(),
VecCondTy->getScalarType(), VecPred,
- CostKind, I);
+ CostKind, Op1Info, Op2Info, I);
}
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
@@ -1077,8 +1076,8 @@ InstructionCost ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
if (ST->hasMVEIntegerOps() && ValTy->isVectorTy())
BaseCost = ST->getMVEVectorCostFactor(CostKind);
- return BaseCost *
- BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return BaseCost * BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
+ CostKind, Op1Info, Op2Info, I);
}
InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 528f082dde32c6..7be53c4bcaa295 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -239,10 +239,12 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 9d0db33d9a1fdc..bf0bef3a2b2f98 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -44,15 +44,17 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
return TTI::TCC_Basic;
}
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const llvm::Instruction *I = nullptr) {
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const llvm::Instruction *I = nullptr) {
if (Opcode == Instruction::Select)
return SCEVCheapExpansionBudget.getValue();
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
}
InstructionCost getArithmeticInstrCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index f47fcff5d60259..bbb9d065b62435 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -255,11 +255,10 @@ InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
CostKind);
}
-InstructionCost HexagonTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
if (ValTy->isVectorTy() && CostKind == TTI::TCK_RecipThroughput) {
if (!isHVXVectorType(ValTy) && ValTy->isFPOrFPVectorTy())
return InstructionCost::getMax();
@@ -267,7 +266,8 @@ InstructionCost HexagonTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
if (Opcode == Instruction::FCmp)
return LT.first + FloatFactor * getTypeNumElements(ValTy);
}
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
}
InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 4a1cfe03d48a74..826644d08d1ac0 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -132,10 +132,12 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b7bdbeb535d526..ec3d3dbc8f6aa4 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -655,18 +655,17 @@ InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
return Cost;
}
-InstructionCost PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost PPCTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
InstructionCost CostFactor =
vectorCostAdjustmentFactor(Opcode, ValTy, nullptr);
if (!CostFactor.isValid())
return InstructionCost::getMax();
- InstructionCost Cost =
- BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ InstructionCost Cost = BaseT::getCmpSelInstrCost(
+ Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
return Cost;
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 126ccb2b3096ea..3cb60d7a1785ae 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -118,10 +118,12 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
const Instruction *I = nullptr);
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 768df71715fa63..43deca67c5693d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1618,23 +1618,38 @@ InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
}
-InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
// Skip if scalar size of ValTy is bigger than ELEN.
if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
+
+ auto getConstantMatCost =
+ [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
+ if (OpInfo.isUniform())
+ // We return 0 we currently ignore the cost of materializing scalar
+ // constants in GPRs.
+ return 0;
+
+ return getConstantPoolLoadCost(ValTy, CostKind);
+ };
+
+ InstructionCost ConstantMatCost;
+ if (Op1Info.isConstant())
+ ConstantMatCost += getConstantMatCost(Op1Info);
+ if (Op2Info.isConstant())
+ ConstantMatCost += getConstantMatCost(Op2Info);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
if (Opcode == Instruction::Select && ValTy->isVectorTy()) {
@@ -1643,14 +1658,16 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// vmandn.mm v8, v8, v9
// vmand.mm v9, v0, v9
// vmor.mm v0, v9, v8
- return LT.first *
- getRISCVInstructionCost(
- {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
- LT.second, CostKind);
+ return ConstantMatCost +
+ LT.first *
+ getRISCVInstructionCost(
+ {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
+ LT.second, CostKind);
}
// vselect and max/min are supported natively.
- return LT.first *
- getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second, CostKind);
+ return ConstantMatCost +
+ LT.first * getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second,
+ CostKind);
}
if (ValTy->getScalarSizeInBits() == 1) {
@@ -1660,7 +1677,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// vmand.mm v9, v0, v9
// vmor.mm v0, v9, v8
MVT InterimVT = LT.second.changeVectorElementType(MVT::i8);
- return LT.first *
+ return ConstantMatCost +
+ LT.first *
getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI},
InterimVT, CostKind) +
LT.first * getRISCVInstructionCost(
@@ -1671,7 +1689,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// vmv.v.x v10, a0
// vmsne.vi v0, v10, 0
// vmerge.vvm v8, v9, v8, v0
- return LT.first * getRISCVInstructionCost(
+ return ConstantMatCost +
+ LT.first * getRISCVInstructionCost(
{RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
LT.second, CostKind);
}
@@ -1680,8 +1699,9 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
CmpInst::isIntPredicate(VecPred)) {
// Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
// provided they incur the same cost across all implementations
- return LT.first *
- getRISCVInstructionCost(RISCV::VMSLT_VV, LT.second, CostKind);
+ return ConstantMatCost + LT.first * getRISCVInstructionCost(RISCV::VMSLT_VV,
+ LT.second,
+ CostKind);
}
if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
@@ -1689,7 +1709,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
- return getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind);
+ return ConstantMatCost +
+ getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind);
// If we do not support the input floating point vector type, use the base
// one which will calculate as:
@@ -1699,7 +1720,7 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
(ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
(ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
// Assuming vector fp compare and mask instructions are all the same cost
// until a need arises to differentiate them.
@@ -1708,7 +1729,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
- return LT.first * getRISCVInstructionCost(
+ return ConstantMatCost +
+ LT.first * getRISCVInstructionCost(
{RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
LT.second, CostKind);
@@ -1716,9 +1738,10 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
- return LT.first *
- getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM},
- LT.second, CostKind);
+ return ConstantMatCost +
+ LT.first *
+ getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM},
+ LT.second, CostKind);
case CmpInst::FCMP_OEQ: // vmfeq.vv
case CmpInst::FCMP_OGT: // vmflt.vv
@@ -1726,8 +1749,9 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
case CmpInst::FCMP_OLT: // vmflt.vv
case CmpInst::FCMP_OLE: // vmfle.vv
case CmpInst::FCMP_UNE: // vmfne.vv
- return LT.first *
- getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind);
+ return ConstantMatCost +
+ LT.first *
+ getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind);
default:
break;
}
@@ -1750,7 +1774,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// TODO: Add cost for scalar type.
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
}
InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index f16c4fc0eed023..65bbd905508557 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -200,10 +200,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
TTI::OperandValueInfo OpdInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 3cd1e05aa5d18c..e44777c5c48575 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -959,13 +959,13 @@ static unsigned getOperandsExtensionCost(const Instruction *I) {
return ExtCost;
}
-InstructionCost SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost SystemZTTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info);
if (!ValTy->isVectorTy()) {
switch (Opcode) {
@@ -1041,7 +1041,8 @@ InstructionCost SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
}
}
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info);
}
InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 9294fada1eb77a..e221200cfa08c4 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -104,10 +104,12 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0fa138cefc3b86..46bc73c5e928e0 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3157,15 +3157,14 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
}
-InstructionCost X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I) {
+InstructionCost X86TTIImpl::getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+ TTI::OperandValueInfo Op2Info, const Instruction *I) {
// Early out if this type isn't scalar/vector integer/float.
if (!(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy()))
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- I);
+ Op1Info, Op2Info, I);
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
@@ -3229,9 +3228,11 @@ InstructionCost X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
// Use FCMP_UEQ expansion - FCMP_ONE should be the same.
if (CondTy && !ST->hasAVX())
return getCmpSelInstrCost(Opcode, ValTy, CondTy,
- CmpInst::Predicate::FCMP_UNO, CostKind) +
+ CmpInst::Predicate::FCMP_UNO, CostKind,
+ Op1Info, Op2Info) +
getCmpSelInstrCost(Opcode, ValTy, CondTy,
- CmpInst::Predicate::FCMP_OEQ, CostKind) +
+ CmpInst::Predicate::FCMP_OEQ, CostKind,
+ Op1Info, Op2Info) +
getArithmeticInstrCost(Instruction::Or, CondTy, CostKind);
break;
@@ -3451,7 +3452,8 @@ InstructionCost X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
if (ValTy->getScalarType()->isFloatingPointTy())
return 3;
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ Op1Info, Op2Info, I);
}
unsigned X86TTIImpl::getAtomicMemIntrinsicMaxElementSize() const { return 16; }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 8ea67dcbe5166c..c16461b157e07f 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -156,10 +156,12 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
- InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
- CmpInst::Predicate VecPred,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr);
+ InstructionCost getCmpSelInstrCost(
+ unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
+ TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d296511b977992..c6507fcc2813b3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6589,7 +6589,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
Pred = Cmp->getPredicate();
return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, Pred,
- CostKind, I);
+ CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, I);
}
case Instruction::ICmp:
case Instruction::FCmp: {
@@ -6608,7 +6609,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
VectorTy = ToVectorTy(ValTy, VF);
return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr,
cast<CmpInst>(I)->getPredicate(), CostKind,
- I);
+ {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, I);
}
case Instruction::Store:
case Instruction::Load: {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e3dbe6260983e..02456b3ce0c175 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -10243,9 +10243,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (VI && SelectOnly) {
assert(!Ty->isVectorTy() && "Expected only for scalar type.");
auto *CI = cast<CmpInst>(VI->getOperand(0));
- IntrinsicCost -=
- TTI->getCmpSelInstrCost(CI->getOpcode(), Ty, Builder.getInt1Ty(),
- CI->getPredicate(), CostKind, CI);
+ IntrinsicCost -= TTI->getCmpSelInstrCost(
+ CI->getOpcode(), Ty, Builder.getInt1Ty(), CI->getPredicate(),
+ CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, CI);
}
return IntrinsicCost;
};
@@ -10509,7 +10510,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InstructionCost ScalarCost = TTI->getCmpSelInstrCost(
E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred,
- CostKind, VI);
+ CostKind, getOperandInfo(VI->getOperand(0)),
+ getOperandInfo(VI->getOperand(1)), VI);
InstructionCost IntrinsicCost = GetMinMaxCost(OrigScalarTy, VI);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
@@ -10519,8 +10521,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto GetVectorCost = [&](InstructionCost CommonCost) {
auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
- InstructionCost VecCost = TTI->getCmpSelInstrCost(
- E->getOpcode(), VecTy, MaskTy, VecPred, CostKind, VL0);
+ InstructionCost VecCost =
+ TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy, VecPred,
+ CostKind, getOperandInfo(E->getOperand(0)),
+ getOperandInfo(E->getOperand(1)), VL0);
if (auto *SI = dyn_cast<SelectInst>(VL0)) {
auto *CondType =
getWidenedType(SI->getCondition()->getType(), VL.size());
@@ -10760,11 +10764,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
TTIRef.getArithmeticInstrCost(E->getAltOpcode(), VecTy, CostKind);
} else if (auto *CI0 = dyn_cast<CmpInst>(VL0)) {
auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
- VecCost = TTIRef.getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy,
- CI0->getPredicate(), CostKind, VL0);
+ VecCost = TTIRef.getCmpSelInstrCost(
+ E->getOpcode(), VecTy, MaskTy, CI0->getPredicate(), CostKind,
+ {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
+ VL0);
VecCost += TTIRef.getCmpSelInstrCost(
E->getOpcode(), VecTy, MaskTy,
cast<CmpInst>(E->getAltOp())->getPredicate(), CostKind,
+ {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
E->getAltOp());
} else {
Type *SrcSclTy = E->getMainOp()->getOperand(0)->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f33293e65010f9..7041fb638ecac3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1267,7 +1267,9 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
Type *VectorTy = ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
return Ctx.TTI.getCmpSelInstrCost(Opcode, VectorTy, nullptr, getPredicate(),
- CostKind, CtxI);
+ CostKind,
+ {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
}
default:
llvm_unreachable("Unsupported opcode for instruction");
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
index 9eadcaca6bb55e..2bf1e5d26e2da9 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
@@ -394,10 +394,10 @@ define void @select() {
define void @select_of_constants() {
; CHECK-LABEL: 'select_of_constants'
; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = select i1 undef, <2 x i64> <i64 128, i64 128>, <2 x i64> zeroinitializer
-; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = select i1 undef, <2 x i64> <i64 128, i64 127>, <2 x i64> zeroinitializer
-; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = select i1 undef, <2 x i64> <i64 0, i64 1>, <2 x i64> zeroinitializer
-; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = select i1 undef, <2 x i64> <i64 128, i64 533>, <2 x i64> <i64 0, i64 573>
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = select <4 x i1> undef, <4 x i32> <i32 524288, i32 262144, i32 131072, i32 65536>, <4 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %2 = select i1 undef, <2 x i64> <i64 128, i64 127>, <2 x i64> zeroinitializer
+; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %3 = select i1 undef, <2 x i64> <i64 0, i64 1>, <2 x i64> zeroinitializer
+; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %4 = select i1 undef, <2 x i64> <i64 128, i64 533>, <2 x i64> <i64 0, i64 573>
+; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = select <4 x i1> undef, <4 x i32> <i32 524288, i32 262144, i32 131072, i32 65536>, <4 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; Splat constants
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
index 4496b19fa200c5..9cfc5f86cb014a 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
@@ -31,13 +31,17 @@ define i32 @pow2_zero_constant_shift(i16 zeroext %a, i16 zeroext %b, i16 zeroext
define i32 @pow2_zero_variable_shift(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i16 zeroext %d) {
; CHECK-LABEL: define i32 @pow2_zero_variable_shift(
; CHECK-SAME: i16 zeroext [[A:%.*]], i16 zeroext [[B:%.*]], i16 zeroext [[C:%.*]], i16 zeroext [[D:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[B]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> [[TMP2]], i16 [[C]], i32 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i16> [[TMP3]], i16 [[D]], i32 3
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[TMP4]], <i16 1, i16 1, i16 1, i16 1>
-; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> <i32 524288, i32 262144, i32 131072, i32 65536>, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[OR_RDX2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP6]])
+; CHECK-NEXT: [[T39_I0:%.*]] = icmp eq i16 [[A]], 1
+; CHECK-NEXT: [[T39_I1:%.*]] = icmp eq i16 [[B]], 1
+; CHECK-NEXT: [[T39_I2:%.*]] = icmp eq i16 [[C]], 1
+; CHECK-NEXT: [[T39_I3:%.*]] = icmp eq i16 [[D]], 1
+; CHECK-NEXT: [[T40_I0:%.*]] = select i1 [[T39_I0]], i32 524288, i32 0
+; CHECK-NEXT: [[T40_I1:%.*]] = select i1 [[T39_I1]], i32 262144, i32 0
+; CHECK-NEXT: [[T40_I2:%.*]] = select i1 [[T39_I2]], i32 131072, i32 0
+; CHECK-NEXT: [[T40_I3:%.*]] = select i1 [[T39_I3]], i32 65536, i32 0
+; CHECK-NEXT: [[OR_RDX0:%.*]] = or i32 [[T40_I0]], [[T40_I1]]
+; CHECK-NEXT: [[OR_RDX1:%.*]] = or i32 [[T40_I2]], [[T40_I3]]
+; CHECK-NEXT: [[OR_RDX2:%.*]] = or i32 [[OR_RDX0]], [[OR_RDX1]]
; CHECK-NEXT: ret i32 [[OR_RDX2]]
;
%t39.i0 = icmp eq i16 %a, 1
>From 398e69ff7e4e18f4c85db02c7c398123d92f5355 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Tue, 24 Sep 2024 11:46:41 -0700
Subject: [PATCH 2/3] Address review comment
---
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 43deca67c5693d..0a5ea76e5a4f35 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1635,7 +1635,7 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
- auto getConstantMatCost =
+ auto GetConstantMatCost =
[&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
if (OpInfo.isUniform())
// We return 0 we currently ignore the cost of materializing scalar
@@ -1647,9 +1647,9 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
InstructionCost ConstantMatCost;
if (Op1Info.isConstant())
- ConstantMatCost += getConstantMatCost(Op1Info);
+ ConstantMatCost += GetConstantMatCost(Op1Info);
if (Op2Info.isConstant())
- ConstantMatCost += getConstantMatCost(Op2Info);
+ ConstantMatCost += GetConstantMatCost(Op2Info);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
if (Opcode == Instruction::Select && ValTy->isVectorTy()) {
>From 4d9a543d15b6e93e363a76a194b3e7ce59480dee Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Tue, 24 Sep 2024 15:55:18 -0700
Subject: [PATCH 3/3] Backout SLP changes
---
.../lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 ++++----
.../RISCV/select-profitability.ll | 18 +++++++-----------
2 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 02456b3ce0c175..2bf22f7c6d79ad 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -10510,8 +10510,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InstructionCost ScalarCost = TTI->getCmpSelInstrCost(
E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred,
- CostKind, getOperandInfo(VI->getOperand(0)),
- getOperandInfo(VI->getOperand(1)), VI);
+ CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, VI);
InstructionCost IntrinsicCost = GetMinMaxCost(OrigScalarTy, VI);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
@@ -10523,8 +10523,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InstructionCost VecCost =
TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy, VecPred,
- CostKind, getOperandInfo(E->getOperand(0)),
- getOperandInfo(E->getOperand(1)), VL0);
+ CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+ {TTI::OK_AnyValue, TTI::OP_None}, VL0);
if (auto *SI = dyn_cast<SelectInst>(VL0)) {
auto *CondType =
getWidenedType(SI->getCondition()->getType(), VL.size());
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
index 9cfc5f86cb014a..4496b19fa200c5 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/select-profitability.ll
@@ -31,17 +31,13 @@ define i32 @pow2_zero_constant_shift(i16 zeroext %a, i16 zeroext %b, i16 zeroext
define i32 @pow2_zero_variable_shift(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i16 zeroext %d) {
; CHECK-LABEL: define i32 @pow2_zero_variable_shift(
; CHECK-SAME: i16 zeroext [[A:%.*]], i16 zeroext [[B:%.*]], i16 zeroext [[C:%.*]], i16 zeroext [[D:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[T39_I0:%.*]] = icmp eq i16 [[A]], 1
-; CHECK-NEXT: [[T39_I1:%.*]] = icmp eq i16 [[B]], 1
-; CHECK-NEXT: [[T39_I2:%.*]] = icmp eq i16 [[C]], 1
-; CHECK-NEXT: [[T39_I3:%.*]] = icmp eq i16 [[D]], 1
-; CHECK-NEXT: [[T40_I0:%.*]] = select i1 [[T39_I0]], i32 524288, i32 0
-; CHECK-NEXT: [[T40_I1:%.*]] = select i1 [[T39_I1]], i32 262144, i32 0
-; CHECK-NEXT: [[T40_I2:%.*]] = select i1 [[T39_I2]], i32 131072, i32 0
-; CHECK-NEXT: [[T40_I3:%.*]] = select i1 [[T39_I3]], i32 65536, i32 0
-; CHECK-NEXT: [[OR_RDX0:%.*]] = or i32 [[T40_I0]], [[T40_I1]]
-; CHECK-NEXT: [[OR_RDX1:%.*]] = or i32 [[T40_I2]], [[T40_I3]]
-; CHECK-NEXT: [[OR_RDX2:%.*]] = or i32 [[OR_RDX0]], [[OR_RDX1]]
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[A]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[B]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> [[TMP2]], i16 [[C]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i16> [[TMP3]], i16 [[D]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[TMP4]], <i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> <i32 524288, i32 262144, i32 131072, i32 65536>, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[OR_RDX2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP6]])
; CHECK-NEXT: ret i32 [[OR_RDX2]]
;
%t39.i0 = icmp eq i16 %a, 1
More information about the llvm-commits
mailing list