[llvm-branch-commits] [llvm] [TTI][SLP] Pass the scalars of the lanes to getArithmeticInstrCost (PR #241758)

Alexey Bataev via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sat Nov 21 13:57:04 PST 2026


https://github.com/alexey-bataev commented:

ISD::FDIV/ISD::FREM afn check skips copyable FPMathOperator lanes that strip afn during vectorization
```
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -541,19 +541,13 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
 
   MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
 
-  // A vector operation has a property of its context instruction only if all
-  // lanes of the same opcode have it.
-  auto AllLanes = [&](function_ref<bool(const Instruction *)> Pred,
-                      bool SkipOtherOpcodes) {
+  // A vector operation has a property of its context instruction only if all
+  // non-poison lanes satisfy Pred.
+  auto AllLanes = [&](function_ref<bool(const Value *)> Pred) {
     if (!CtxI || CtxI->getOpcode() != Opcode || !Pred(CtxI))
       return false;
     return all_of(Scalars, [&](const Value *V) {
-      if (!V || isa<PoisonValue>(V))
-        return true;
-      const auto *I = dyn_cast<Instruction>(V);
-      if (!I || I->getOpcode() != Opcode)
-        return SkipOtherOpcodes;
-      return I == CtxI || Pred(I);
+      return !V || isa<PoisonValue>(V) || V == CtxI || Pred(V);
     });
   };
 
@@ -607,18 +601,17 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     if (CtxI &&
         (CtxI->getType() == Ty ||
          getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1) &&
-        AllLanes(
-            [&](const Instruction *FMul) {
-              if (!FMul->hasOneUse())
-                return false;
-              const auto *FAddSub =
-                  dyn_cast<BinaryOperator>(*FMul->user_begin());
-              return FAddSub &&
-                     (FAddSub->getOpcode() == Instruction::FAdd ||
-                      FAddSub->getOpcode() == Instruction::FSub) &&
-                     isFusedFMul(*TLI, Ty, FMul, FAddSub);
-            },
-            /*SkipOtherOpcodes=*/false))
+        AllLanes([&](const Value *V) {
+          const auto *FMul = dyn_cast<Instruction>(V);
+          if (!FMul || FMul->getOpcode() != Opcode || !FMul->hasOneUse())
+            return false;
+          const auto *FAddSub = dyn_cast<BinaryOperator>(*FMul->user_begin());
+          return FAddSub &&
+                 (FAddSub->getOpcode() == Instruction::FAdd ||
+                  FAddSub->getOpcode() == Instruction::FSub) &&
+                 isFusedFMul(*TLI, Ty, FMul, FAddSub);
+        }))
       return TargetTransformInfo::TCC_Free;
     [[fallthrough]];
   case ISD::FADD:
@@ -672,9 +665,10 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
       return LT.first * Cost * NElts;
     }
 
-    if (SLT == MVT::f32 &&
-        AllLanes([](const Instruction *I) { return I->hasApproxFunc(); },
-                 /*SkipOtherOpcodes=*/true)) {
+    if (SLT == MVT::f32 && AllLanes([](const Value *V) {
+          const auto *FPMO = dyn_cast<FPMathOperator>(V);
+          return !FPMO || FPMO->hasApproxFunc();
+        })) {
       // Fast unsafe fdiv lowering:
       // f32 rcp
       // f32 fmul

```
Remaining vector getArithmeticInstrCost calls passing single-lane CtxI without Scalars
```
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14316,7 +14316,7 @@ bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
       TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
       TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
                                   TTI::getOperandInfo(LhsTE->Scalars),
                                   TTI::getOperandInfo(RhsTE->Scalars), {},
-                                  TE.getMainOp()) +
+                                  TE.getMainOp(), TLI, TE.Scalars) +
       TTI->getCastInstrCost(
           Instruction::ZExt, VecTy,
@@ -14554,10 +14554,12 @@ bool BoUpSLP::matchesBitPack(const TreeEntry &TE) const {
       TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
       TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
                                   TTI::getOperandInfo(LhsTE->Scalars),
                                   TTI::getOperandInfo(ShiftTE->Scalars),
-                                  /*Args=*/{}, ShlTE->getMainOp(), TLI);
+                                  /*Args=*/{}, ShlTE->getMainOp(), TLI,
+                                  ShlTE->Scalars);
   if (TE.getOpcode() == Instruction::And)
     VecCost += TTI->getArithmeticInstrCost(
         Instruction::And, VecTy, CostKind, TTI::getOperandInfo(ShlTE->Scalars),
-        TTI::getOperandInfo(MaskTE->Scalars), /*Args=*/{}, TE.getMainOp(), TLI);
+        TTI::getOperandInfo(MaskTE->Scalars), /*Args=*/{}, TE.getMainOp(), TLI,
+        TE.Scalars);

```

https://github.com/llvm/llvm-project/pull/241758


More information about the llvm-branch-commits mailing list