[llvm-branch-commits] [llvm] [TTI][SLP] Pass the scalars of the lanes to getArithmeticInstrCost (PR #241758)
Alexey Bataev via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Nov 21 13:57:04 PST 2026
https://github.com/alexey-bataev commented:
ISD::FDIV/ISD::FREM afn check skips copyable FPMathOperator lanes that strip afn during vectorization
```
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -541,19 +541,13 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
- // A vector operation has a property of its context instruction only if all
- // lanes of the same opcode have it.
- auto AllLanes = [&](function_ref<bool(const Instruction *)> Pred,
- bool SkipOtherOpcodes) {
+ // A vector operation has a property of its context instruction only if all
+ // non-poison lanes satisfy Pred.
+ auto AllLanes = [&](function_ref<bool(const Value *)> Pred) {
if (!CtxI || CtxI->getOpcode() != Opcode || !Pred(CtxI))
return false;
return all_of(Scalars, [&](const Value *V) {
- if (!V || isa<PoisonValue>(V))
- return true;
- const auto *I = dyn_cast<Instruction>(V);
- if (!I || I->getOpcode() != Opcode)
- return SkipOtherOpcodes;
- return I == CtxI || Pred(I);
+ return !V || isa<PoisonValue>(V) || V == CtxI || Pred(V);
});
};
@@ -607,18 +601,17 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
if (CtxI &&
(CtxI->getType() == Ty ||
getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1) &&
- AllLanes(
- [&](const Instruction *FMul) {
- if (!FMul->hasOneUse())
- return false;
- const auto *FAddSub =
- dyn_cast<BinaryOperator>(*FMul->user_begin());
- return FAddSub &&
- (FAddSub->getOpcode() == Instruction::FAdd ||
- FAddSub->getOpcode() == Instruction::FSub) &&
- isFusedFMul(*TLI, Ty, FMul, FAddSub);
- },
- /*SkipOtherOpcodes=*/false))
+ AllLanes([&](const Value *V) {
+ const auto *FMul = dyn_cast<Instruction>(V);
+ if (!FMul || FMul->getOpcode() != Opcode || !FMul->hasOneUse())
+ return false;
+ const auto *FAddSub = dyn_cast<BinaryOperator>(*FMul->user_begin());
+ return FAddSub &&
+ (FAddSub->getOpcode() == Instruction::FAdd ||
+ FAddSub->getOpcode() == Instruction::FSub) &&
+ isFusedFMul(*TLI, Ty, FMul, FAddSub);
+ }))
return TargetTransformInfo::TCC_Free;
[[fallthrough]];
case ISD::FADD:
@@ -672,9 +665,10 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
return LT.first * Cost * NElts;
}
- if (SLT == MVT::f32 &&
- AllLanes([](const Instruction *I) { return I->hasApproxFunc(); },
- /*SkipOtherOpcodes=*/true)) {
+ if (SLT == MVT::f32 && AllLanes([](const Value *V) {
+ const auto *FPMO = dyn_cast<FPMathOperator>(V);
+ return !FPMO || FPMO->hasApproxFunc();
+ })) {
// Fast unsafe fdiv lowering:
// f32 rcp
// f32 fmul
```
Remaining vector getArithmeticInstrCost calls passing single-lane CtxI without Scalars
```
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14316,7 +14316,7 @@ bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
TTI::getOperandInfo(LhsTE->Scalars),
TTI::getOperandInfo(RhsTE->Scalars), {},
- TE.getMainOp()) +
+ TE.getMainOp(), TLI, TE.Scalars) +
TTI->getCastInstrCost(
Instruction::ZExt, VecTy,
@@ -14554,10 +14554,12 @@ bool BoUpSLP::matchesBitPack(const TreeEntry &TE) const {
TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
TTI::getOperandInfo(LhsTE->Scalars),
TTI::getOperandInfo(ShiftTE->Scalars),
- /*Args=*/{}, ShlTE->getMainOp(), TLI);
+ /*Args=*/{}, ShlTE->getMainOp(), TLI,
+ ShlTE->Scalars);
if (TE.getOpcode() == Instruction::And)
VecCost += TTI->getArithmeticInstrCost(
Instruction::And, VecTy, CostKind, TTI::getOperandInfo(ShlTE->Scalars),
- TTI::getOperandInfo(MaskTE->Scalars), /*Args=*/{}, TE.getMainOp(), TLI);
+ TTI::getOperandInfo(MaskTE->Scalars), /*Args=*/{}, TE.getMainOp(), TLI,
+ TE.Scalars);
```
https://github.com/llvm/llvm-project/pull/241758
More information about the llvm-branch-commits
mailing list