[llvm-branch-commits] [llvm] [TTI][SLP] Pass the scalars of the lanes to getArithmeticInstrCost (PR #241758)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Fri Nov 20 00:30:34 PST 2026


https://github.com/michaelselehov updated https://github.com/llvm/llvm-project/pull/241758

>From 25366511a92bf7bc3f4a3e83d4f353b7bcc034bd Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Thu, 12 Nov 2026 05:54:43 -0600
Subject: [PATCH 1/4] [TTI][SLP] Pass the scalars of the lanes to
 getArithmeticInstrCost

SLP prices a vector instruction with one scalar lane as the context. A
target that reads a property of the context, like the users of an fmul
that fuses into an fma, gives the whole vector the property of that lane.

Add the scalars of the lanes to getArithmeticInstrCost, like
getAltInstrCost does, and pass them from SLP. AMDGPU uses them for the
fmul fusion and the fdiv afn costs: the vector gets the cheaper cost only
if all lanes qualify.

Assisted-by: Claude Opus
---
 .../llvm/Analysis/TargetTransformInfo.h       |  6 ++-
 .../llvm/Analysis/TargetTransformInfoImpl.h   |  3 +-
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |  9 ++--
 llvm/lib/Analysis/TargetTransformInfo.cpp     |  4 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |  5 ++-
 .../AArch64/AArch64TargetTransformInfo.h      |  4 +-
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      | 42 +++++++++++++------
 .../Target/AMDGPU/AMDGPUTargetTransformInfo.h |  4 +-
 .../lib/Target/ARM/ARMTargetTransformInfo.cpp |  3 +-
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h  |  4 +-
 llvm/lib/Target/BPF/BPFTargetTransformInfo.h  |  4 +-
 .../Hexagon/HexagonTargetTransformInfo.cpp    |  9 ++--
 .../Hexagon/HexagonTargetTransformInfo.h      |  4 +-
 .../Target/Lanai/LanaiTargetTransformInfo.h   |  4 +-
 .../LoongArchTargetTransformInfo.cpp          |  5 ++-
 .../LoongArch/LoongArchTargetTransformInfo.h  |  4 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.cpp |  3 +-
 .../Target/NVPTX/NVPTXTargetTransformInfo.h   |  4 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |  7 ++--
 .../Target/PowerPC/PPCTargetTransformInfo.h   |  4 +-
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 13 +++---
 .../Target/RISCV/RISCVTargetTransformInfo.h   |  4 +-
 .../SystemZ/SystemZTargetTransformInfo.cpp    |  9 ++--
 .../SystemZ/SystemZTargetTransformInfo.h      |  4 +-
 .../WebAssemblyTargetTransformInfo.cpp        |  3 +-
 .../WebAssemblyTargetTransformInfo.h          |  4 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  5 ++-
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |  4 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  4 +-
 .../AMDGPU/fdiv-afn-ctx-lanes.ll              | 21 ++++++----
 .../AMDGPU/fmul-fusion-ctx-mixed-lanes.ll     | 28 +++++++++----
 31 files changed, 142 insertions(+), 89 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index d72ba45d962055..6163ed8350143a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1635,12 +1635,16 @@ class TargetTransformInfo {
   /// provide even more information.
   /// \p TLibInfo is used to search for platform specific vector library
   /// functions for instructions that might be converted to calls (e.g. frem).
+  /// \p Scalars are the scalar instructions of the lanes of a vector operation,
+  /// if known. The target may use them for the properties that differ between
+  /// the lanes, where \p CtxI shows only one lane.
   LLVM_ABI InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
-      const TargetLibraryInfo *TLibInfo = nullptr) const;
+      const TargetLibraryInfo *TLibInfo = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const;
 
   /// Returns the cost estimation for alternating opcode pattern that can be
   /// lowered to a single instruction on the target. In X86 this is for the
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index cbe66ccbc55788..8a8401f3af4fa5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -751,7 +751,8 @@ class LLVM_ABI TargetTransformInfoImplBase {
   virtual InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
-      ArrayRef<const Value *> Args, const Instruction *CtxI = nullptr) const {
+      ArrayRef<const Value *> Args, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const {
     // Widenable conditions will eventually lower into constants, so some
     // operations with them will be trivially optimized away.
     auto IsWidenableCondition = [](const Value *V) {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 2de9b66470f23a..f86b61de1d5176 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1082,8 +1082,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override {
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override {
     // Check if any of the operands are vector operands.
     const TargetLoweringBase *TLI = getTLI();
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -1091,9 +1091,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
     // TODO: Handle more cost kinds.
     if (CostKind != TTI::TCK_RecipThroughput)
-      return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
-                                           Opd1Info, Opd2Info,
-                                           Args, CtxI);
+      return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
+                                           Opd2Info, Args, CtxI, Scalars);
 
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 0a225409bf2536..0490d20c86a358 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1098,7 +1098,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     OperandValueInfo Op1Info, OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    const TargetLibraryInfo *TLibInfo) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // Use call cost for frem instructions that have platform specific vector math
   // functions, as those will be replaced with calls later by SelectionDAG or
@@ -1113,7 +1113,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
   }
 
   InstructionCost Cost = TTIImpl->getArithmeticInstrCost(
-      Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI);
+      Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI, Scalars);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 57d7fe3a924a42..2a5401ea57cca4 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5183,7 +5183,8 @@ std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
 InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
@@ -5224,7 +5225,7 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
       ISD == ISD::FDIV || ISD == ISD::FREM || ISD == ISD::FNEG)
     if (CostKind != TTI::TCK_RecipThroughput)
       return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
-                                           Op2Info, Args, CtxI);
+                                           Op2Info, Args, CtxI, Scalars);
 
   if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
       ISD == ISD::FDIV || ISD == ISD::FREM) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 6907029b6912a8..df47c12f669def 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -253,8 +253,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
   getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 9406c99d550bf5..ea583f0f7c5464 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -527,7 +527,8 @@ static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty,
 InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
@@ -540,6 +541,19 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
 
   MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
 
+  // A vector operation has a property of its context instruction only if all
+  // lanes of the same opcode have it.
+  auto AllLanes = [&](function_ref<bool(const Instruction *)> Pred) {
+    if (!CtxI)
+      return false;
+    if (Scalars.empty())
+      return Pred(CtxI);
+    return all_of(Scalars, [&](const Value *V) {
+      const auto *I = dyn_cast<Instruction>(V);
+      return !I || I->getOpcode() != CtxI->getOpcode() || Pred(I);
+    });
+  };
+
   switch (ISD) {
   case ISD::SHL:
   case ISD::SRL:
@@ -587,16 +601,19 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
     // fused operation. A vector fmul priced with a scalar context fuses with
     // the scalar fadd|fsub only when it is emitted lane by lane.
-    if (CtxI && CtxI->hasOneUse() &&
+    if (CtxI &&
         (CtxI->getType() == Ty ||
-         getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1)) {
-      const auto *FAddSub = dyn_cast<BinaryOperator>(*CtxI->user_begin());
-      if (FAddSub &&
-          (FAddSub->getOpcode() == Instruction::FAdd ||
-           FAddSub->getOpcode() == Instruction::FSub) &&
-          isFusedFMul(*TLI, Ty, CtxI, FAddSub))
-        return TargetTransformInfo::TCC_Free;
-    }
+         getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1) &&
+        AllLanes([&](const Instruction *FMul) {
+          if (!FMul->hasOneUse())
+            return false;
+          const auto *FAddSub = dyn_cast<BinaryOperator>(*FMul->user_begin());
+          return FAddSub &&
+                 (FAddSub->getOpcode() == Instruction::FAdd ||
+                  FAddSub->getOpcode() == Instruction::FSub) &&
+                 isFusedFMul(*TLI, Ty, FMul, FAddSub);
+        }))
+      return TargetTransformInfo::TCC_Free;
     [[fallthrough]];
   case ISD::FADD:
   case ISD::FSUB:
@@ -649,7 +666,8 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
       return LT.first * Cost * NElts;
     }
 
-    if (SLT == MVT::f32 && (CtxI && CtxI->hasApproxFunc())) {
+    if (SLT == MVT::f32 &&
+        AllLanes([](const Instruction *I) { return I->hasApproxFunc(); })) {
       // Fast unsafe fdiv lowering:
       // f32 rcp
       // f32 fmul
@@ -679,7 +697,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
   }
 
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI);
+                                       Args, CtxI, Scalars);
 }
 
 // Return true if there's a potential benefit from using v2f16/v2i16
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 61af5d935c07b8..71a39f10227f20 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -172,8 +172,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 1461ce32cb27c6..b8391ea7603f36 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1393,7 +1393,8 @@ InstructionCost ARMTTIImpl::getShuffleCost(
 InstructionCost ARMTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
   int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
   if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) {
     // Make operations on i1 relatively expensive as this often involves
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index f2b94c5b3990f4..33145cb4110c48 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -237,8 +237,8 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getMemoryOpCost(
       unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 0d5ff2d36aca94..f13b6bbb67b1ac 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -62,8 +62,8 @@ class BPFTTIImpl final : public BasicTTIImplBase<BPFTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override {
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     if (ISD == ISD::ADD && CostKind == TTI::TCK_RecipThroughput)
       return SCEVCheapExpansionBudget.getValue() + 1;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 60cf35e1e90900..21ac5e3d292103 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -269,11 +269,12 @@ InstructionCost HexagonTTIImpl::getCmpSelInstrCost(
 InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
-    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
-                                         Op2Info, Args, CtxI);
+    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+                                         Args, CtxI, Scalars);
 
   if (Ty->isVectorTy()) {
     if (!isHVXVectorType(Ty) && Ty->isFPOrFPVectorTy())
@@ -283,7 +284,7 @@ InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
       return LT.first + FloatFactor * getTypeNumElements(Ty);
   }
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI);
+                                       Args, CtxI, Scalars);
 }
 
 InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 983d94c30eb40c..cc1940f6f314be 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -145,8 +145,8 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                    TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 78f8eb62cb7893..4a02673446a07e 100644
--- a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -95,8 +95,8 @@ class LanaiTTIImpl final : public BasicTTIImplBase<LanaiTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override {
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
     switch (ISD) {
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index ec1f33ae2c5235..9ca6fd4e4ea500 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -171,7 +171,8 @@ LoongArchTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
 InstructionCost LoongArchTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CxtI) const {
+    ArrayRef<const Value *> Args, const Instruction *CxtI,
+    ArrayRef<const Value *> Scalars) const {
 
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -456,7 +457,7 @@ InstructionCost LoongArchTTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CxtI);
+                                       Args, CxtI, Scalars);
 }
 
 // Returns true if Ty is a fixed vector type held in a single LSX/LASX register.
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index a61a339a1fe7d6..ca4e20c6b53dd8 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -58,8 +58,8 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CxtI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   using BaseT::getVectorInstrCost;
   InstructionCost
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index a82f6f3160a9fc..b575edbe29e1b1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -512,7 +512,8 @@ NVPTXTTIImpl::getInstructionCost(const User *U,
 InstructionCost NVPTXTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index a9c8964c55eac3..c0bf88ee4cf434 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -126,8 +126,8 @@ class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
   getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 83fa8c5c10c36e..3e2ea4873cc278 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -599,7 +599,8 @@ InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode,
 InstructionCost PPCTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
   InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr);
@@ -608,8 +609,8 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
-    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
-                                         Op2Info, Args, CtxI);
+    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+                                         Args, CtxI, Scalars);
 
   // Fallback to the default implementation.
   InstructionCost Cost = BaseT::getArithmeticInstrCost(
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 3702283c543408..8e9643f011cdb4 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -109,8 +109,8 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
                  TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index df9cf4a7a482af..8dc7ddc61fba75 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -3090,21 +3090,22 @@ RISCVTTIImpl::getCombinedArithmeticInstructionCost(
 InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI);
+                                         Args, CtxI, Scalars);
 
   if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI);
+                                         Args, CtxI, Scalars);
 
   // Skip if scalar size of Ty is bigger than ELEN.
   if (isa<VectorType>(Ty) && Ty->getScalarSizeInBits() > ST->getELen())
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI);
+                                         Args, CtxI, Scalars);
 
   if (std::optional<InstructionCost> CombinedCost =
           getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Op1Info,
@@ -3134,7 +3135,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
         return Entry->Cost * LT.first;
 
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI);
+                                         Args, CtxI, Scalars);
   }
 
   // f16 with zvfhmin and bf16 will be promoted to f32.
@@ -3227,7 +3228,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     // differentiate them.
     return CastCost + ConstantMatCost +
            BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI);
+                                         Args, CtxI, Scalars);
   }
 
   InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 12921cb17eadd8..99606c1dec9b00 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -288,8 +288,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   bool isElementTypeLegalForScalableVector(Type *Ty) const override {
     return TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, Ty));
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 06f84c78a504d0..29c57eb3088467 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -611,12 +611,13 @@ static bool isFoldableRMW(const Instruction *I, Type *Ty) {
 InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
-    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
-                                         Op2Info, Args, CtxI);
+    return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+                                         Args, CtxI, Scalars);
   if (CtxI && Ty && !Ty->isVectorTy() && isFoldableRMW(CtxI, Ty))
     return TTI::TCC_Free;
   // TODO: return a good value for BB-VECTORIZER that includes the
@@ -791,7 +792,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI);
+                                       Args, CtxI, Scalars);
 }
 
 InstructionCost SystemZTTIImpl::getShuffleCost(
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 8236790a6e2717..d3a6e7efbef9b8 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -102,8 +102,8 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getPartialReductionCost(
       unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index a6ed2eae0064cb..a73a0045029bd7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -56,7 +56,8 @@ TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
 InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   if (ST->hasSIMD128()) {
     static const CostTblEntry ArithCostTbl[]{
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index bfa83a0312a0f3..75127c7ffd73aa 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -66,8 +66,8 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
   getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 76a11d7c77cc9e..3c444acfc928e2 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -225,7 +225,8 @@ unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF,
 InstructionCost X86TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
-    ArrayRef<const Value *> Args, const Instruction *CtxI) const {
+    ArrayRef<const Value *> Args, const Instruction *CtxI,
+    ArrayRef<const Value *> Scalars) const {
 
   // vXi8 multiplications are always promoted to vXi16.
   // Sub-128-bit types can be extended/packed more efficiently.
@@ -1924,7 +1925,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI);
+                                       Args, CtxI, Scalars);
 }
 
 InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 122edb3312001a..6c29c53ac448b2 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -70,8 +70,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
-      ArrayRef<const Value *> Args = {},
-      const Instruction *CtxI = nullptr) const override;
+      ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
                   const SmallBitVector &OpcodeMask,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 07ccb3de8e5da0..7e08589d091b79 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18092,7 +18092,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(E->getOperand(OpIdx));
       InstructionCost Cost = TTI->getArithmeticInstrCost(
           ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {},
-          VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
+          VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI, E->Scalars);
       // N columns need N-1 vector combines; price extra columns
       // conservatively, skipping identity-only columns (not combined by
       // codegen).
@@ -18109,7 +18109,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
           Cost += TTI->getArithmeticInstrCost(
               ShuffleOrOp, VecTy, CostKind, {},
               TTI::getOperandInfo(E->getOperand(Idx)), {},
-              VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
+              VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI, E->Scalars);
         }
       }
       return Cost + CommonCost;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-ctx-lanes.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-ctx-lanes.ll
index 8b9554af55bae2..f8953761552fcf 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-ctx-lanes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fdiv-afn-ctx-lanes.ll
@@ -9,17 +9,24 @@ define void @fdiv_mixed(ptr %a, ptr %b, ptr %out) {
 ; CHECK-LABEL: define void @fdiv_mixed(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[OUT:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT:    [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[A]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[B]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv <2 x float> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 2
+; CHECK-NEXT:    [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT:    [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; CHECK-NEXT:    [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT:    [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; CHECK-NEXT:    [[B0:%.*]] = load float, ptr [[B]], align 4
+; CHECK-NEXT:    [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; CHECK-NEXT:    [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; CHECK-NEXT:    [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; CHECK-NEXT:    [[D0:%.*]] = fdiv afn float [[A0]], [[B0]]
+; CHECK-NEXT:    [[D3:%.*]] = fdiv float [[A3]], [[B3]]
+; CHECK-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[AP1]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[BP1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <2 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[OUT]], align 4
+; CHECK-NEXT:    store float [[D0]], ptr [[OUT]], align 4
 ; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[O1]], align 4
+; CHECK-NEXT:    [[O3:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store float [[D3]], ptr [[O3]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-ctx-mixed-lanes.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-ctx-mixed-lanes.ll
index 95e84c99925bcc..2ecff1f6b76881 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-ctx-mixed-lanes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-ctx-mixed-lanes.ll
@@ -9,18 +9,32 @@
 define double @mixed_lanes(ptr noalias %e, ptr %a, ptr %b) {
 ; CHECK-LABEL: define double @mixed_lanes(
 ; CHECK-SAME: ptr noalias [[E:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x double>, ptr [[B]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul reassoc contract <4 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[M1:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT:    [[B0:%.*]] = load double, ptr [[B]], align 8
+; CHECK-NEXT:    [[AP1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[BP1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[AP1]], align 8
+; CHECK-NEXT:    [[B1:%.*]] = load double, ptr [[BP1]], align 8
+; CHECK-NEXT:    [[AP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
+; CHECK-NEXT:    [[BP2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 2
+; CHECK-NEXT:    [[A2:%.*]] = load double, ptr [[AP2]], align 8
+; CHECK-NEXT:    [[B2:%.*]] = load double, ptr [[BP2]], align 8
+; CHECK-NEXT:    [[AP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[BP3:%.*]] = getelementptr inbounds double, ptr [[B]], i64 3
+; CHECK-NEXT:    [[A3:%.*]] = load double, ptr [[AP3]], align 8
+; CHECK-NEXT:    [[B3:%.*]] = load double, ptr [[BP3]], align 8
+; CHECK-NEXT:    [[M0:%.*]] = fmul reassoc contract double [[A0]], [[B0]]
+; CHECK-NEXT:    [[M1:%.*]] = fmul reassoc contract double [[A1]], [[B1]]
+; CHECK-NEXT:    [[M2:%.*]] = fmul reassoc contract double [[A2]], [[B2]]
+; CHECK-NEXT:    [[M3:%.*]] = fmul reassoc contract double [[A3]], [[B3]]
 ; CHECK-NEXT:    store double [[M1]], ptr [[E]], align 8
 ; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds double, ptr [[E]], i64 1
-; CHECK-NEXT:    [[M2:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
 ; CHECK-NEXT:    store double [[M2]], ptr [[E2]], align 8
 ; CHECK-NEXT:    [[E3:%.*]] = getelementptr inbounds double, ptr [[E]], i64 2
-; CHECK-NEXT:    [[M3:%.*]] = extractelement <4 x double> [[TMP3]], i64 3
 ; CHECK-NEXT:    store double [[M3]], ptr [[E3]], align 8
-; CHECK-NEXT:    [[S3:%.*]] = call reassoc contract double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP3]])
+; CHECK-NEXT:    [[S1:%.*]] = fadd reassoc contract double [[M0]], [[M1]]
+; CHECK-NEXT:    [[S2:%.*]] = fadd reassoc contract double [[S1]], [[M2]]
+; CHECK-NEXT:    [[S3:%.*]] = fadd reassoc contract double [[S2]], [[M3]]
 ; CHECK-NEXT:    ret double [[S3]]
 ;
   %a0 = load double, ptr %a

>From f15825d38b98e5e7eff28221f1fc61c0e1f34055 Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Tue, 17 Nov 2026 00:02:50 -0600
Subject: [PATCH 2/4] [TTI] Pass TLibInfo to the getArithmeticInstrCost
 implementations

---
 llvm/include/llvm/Analysis/TargetTransformInfoImpl.h |  1 +
 llvm/include/llvm/CodeGen/BasicTTIImpl.h             |  8 +++++---
 llvm/lib/Analysis/TargetTransformInfo.cpp            |  2 +-
 .../Target/AArch64/AArch64TargetTransformInfo.cpp    |  5 +++--
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h |  1 +
 llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp |  4 ++--
 llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h   |  1 +
 llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp       |  2 +-
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h         |  1 +
 llvm/lib/Target/BPF/BPFTargetTransformInfo.h         |  1 +
 .../Target/Hexagon/HexagonTargetTransformInfo.cpp    |  6 +++---
 llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h |  1 +
 llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h     |  1 +
 .../LoongArch/LoongArchTargetTransformInfo.cpp       |  4 ++--
 .../Target/LoongArch/LoongArchTargetTransformInfo.h  |  1 +
 llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp   |  2 +-
 llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h     |  1 +
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp   |  4 ++--
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h     |  1 +
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp   | 12 ++++++------
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h     |  1 +
 .../Target/SystemZ/SystemZTargetTransformInfo.cpp    |  6 +++---
 llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h |  1 +
 .../WebAssembly/WebAssemblyTargetTransformInfo.cpp   |  2 +-
 .../WebAssembly/WebAssemblyTargetTransformInfo.h     |  1 +
 llvm/lib/Target/X86/X86TargetTransformInfo.cpp       |  4 ++--
 llvm/lib/Target/X86/X86TargetTransformInfo.h         |  1 +
 27 files changed, 46 insertions(+), 29 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 8a8401f3af4fa5..f0b32878c28ec5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -752,6 +752,7 @@ class LLVM_ABI TargetTransformInfoImplBase {
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
       ArrayRef<const Value *> Args, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const {
     // Widenable conditions will eventually lower into constants, so some
     // operations with them will be trivially optimized away.
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index f86b61de1d5176..bf532f5e87d447 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1083,6 +1083,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Opd2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override {
     // Check if any of the operands are vector operands.
     const TargetLoweringBase *TLI = getTLI();
@@ -1092,7 +1093,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // TODO: Handle more cost kinds.
     if (CostKind != TTI::TCK_RecipThroughput)
       return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info,
-                                           Opd2Info, Args, CtxI, Scalars);
+                                           Opd2Info, Args, CtxI, TLibInfo,
+                                           Scalars);
 
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
@@ -1142,8 +1144,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     // similarly to what getCastInstrCost() does.
     if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
       InstructionCost Cost = thisT()->getArithmeticInstrCost(
-          Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info,
-          Args, CtxI);
+          Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info, Args,
+          CtxI, TLibInfo);
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
       SmallVector<Type *> Tys(Args.size(), Ty);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 0490d20c86a358..56ae0f7a266a59 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1113,7 +1113,7 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
   }
 
   InstructionCost Cost = TTIImpl->getArithmeticInstrCost(
-      Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI, Scalars);
+      Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CtxI, TLibInfo, Scalars);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 2a5401ea57cca4..a06444d6cda2e8 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5184,7 +5184,7 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // The code-generator is currently not able to handle scalable vectors
   // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
@@ -5225,7 +5225,8 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
       ISD == ISD::FDIV || ISD == ISD::FREM || ISD == ISD::FNEG)
     if (CostKind != TTI::TCK_RecipThroughput)
       return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
-                                           Op2Info, Args, CtxI, Scalars);
+                                           Op2Info, Args, CtxI, TLibInfo,
+                                           Scalars);
 
   if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
       ISD == ISD::FDIV || ISD == ISD::FREM) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index df47c12f669def..24bb718d82bcc1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -254,6 +254,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index ea583f0f7c5464..729cf69da1ff99 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -528,7 +528,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
@@ -697,7 +697,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
   }
 
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI, Scalars);
+                                       Args, CtxI, TLibInfo, Scalars);
 }
 
 // Return true if there's a potential benefit from using v2f16/v2i16
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 71a39f10227f20..2b965a8bb95b82 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -173,6 +173,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index b8391ea7603f36..62afb4c33b43a6 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1394,7 +1394,7 @@ InstructionCost ARMTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
   int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
   if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) {
     // Make operations on i1 relatively expensive as this often involves
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 33145cb4110c48..8b9c1f9b03d274 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -238,6 +238,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getMemoryOpCost(
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index f13b6bbb67b1ac..06e3b1fb4eca39 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -63,6 +63,7 @@ class BPFTTIImpl final : public BasicTTIImplBase<BPFTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
     if (ISD == ISD::ADD && CostKind == TTI::TCK_RecipThroughput)
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 21ac5e3d292103..aa59765409e334 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -270,11 +270,11 @@ InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
 
   if (Ty->isVectorTy()) {
     if (!isHVXVectorType(Ty) && Ty->isFPOrFPVectorTy())
@@ -284,7 +284,7 @@ InstructionCost HexagonTTIImpl::getArithmeticInstrCost(
       return LT.first + FloatFactor * getTypeNumElements(Ty);
   }
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI, Scalars);
+                                       Args, CtxI, TLibInfo, Scalars);
 }
 
 InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index cc1940f6f314be..7550517e6b97e8 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -146,6 +146,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
diff --git a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 4a02673446a07e..3583bf7f332440 100644
--- a/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -96,6 +96,7 @@ class LanaiTTIImpl final : public BasicTTIImplBase<LanaiTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override {
     int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index 9ca6fd4e4ea500..e7ef025273153e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -172,7 +172,7 @@ InstructionCost LoongArchTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CxtI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -457,7 +457,7 @@ InstructionCost LoongArchTTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CxtI, Scalars);
+                                       Args, CxtI, TLibInfo, Scalars);
 }
 
 // Returns true if Ty is a fixed vector type held in a single LSX/LASX register.
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index ca4e20c6b53dd8..232723efb8297d 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -59,6 +59,7 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CxtI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   using BaseT::getVectorInstrCost;
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index b575edbe29e1b1..3bcab1ed461ae0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -513,7 +513,7 @@ InstructionCost NVPTXTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index c0bf88ee4cf434..a2775e35be80ba 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -127,6 +127,7 @@ class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3e2ea4873cc278..4b869cd4c0f405 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -600,7 +600,7 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
   InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr);
@@ -610,7 +610,7 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
 
   // Fallback to the default implementation.
   InstructionCost Cost = BaseT::getArithmeticInstrCost(
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 8e9643f011cdb4..7240fc8276d660 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -110,6 +110,7 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 8dc7ddc61fba75..12906d2aee6567 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -3091,21 +3091,21 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
 
   if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
 
   // Skip if scalar size of Ty is bigger than ELEN.
   if (isa<VectorType>(Ty) && Ty->getScalarSizeInBits() > ST->getELen())
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
 
   if (std::optional<InstructionCost> CombinedCost =
           getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Op1Info,
@@ -3135,7 +3135,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
         return Entry->Cost * LT.first;
 
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
   }
 
   // f16 with zvfhmin and bf16 will be promoted to f32.
@@ -3228,7 +3228,7 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     // differentiate them.
     return CastCost + ConstantMatCost +
            BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
   }
 
   InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 99606c1dec9b00..20b1992aaf1234 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -289,6 +289,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   bool isElementTypeLegalForScalableVector(Type *Ty) const override {
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 29c57eb3088467..b24aa1fbe27e66 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -612,12 +612,12 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // TODO: Handle more cost kinds.
   if (CostKind != TTI::TCK_RecipThroughput)
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CtxI, Scalars);
+                                         Args, CtxI, TLibInfo, Scalars);
   if (CtxI && Ty && !Ty->isVectorTy() && isFoldableRMW(CtxI, Ty))
     return TTI::TCC_Free;
   // TODO: return a good value for BB-VECTORIZER that includes the
@@ -792,7 +792,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI, Scalars);
+                                       Args, CtxI, TLibInfo, Scalars);
 }
 
 InstructionCost SystemZTTIImpl::getShuffleCost(
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index d3a6e7efbef9b8..94dba0e0f4e5ef 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -103,6 +103,7 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost getPartialReductionCost(
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index a73a0045029bd7..db247d1c5c21d5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -57,7 +57,7 @@ InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   if (ST->hasSIMD128()) {
     static const CostTblEntry ArithCostTbl[]{
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 75127c7ffd73aa..b883e8e0e83e6c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -67,6 +67,7 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
 
   InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 3c444acfc928e2..281f0808a9f615 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -226,7 +226,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
     ArrayRef<const Value *> Args, const Instruction *CtxI,
-    ArrayRef<const Value *> Scalars) const {
+    const TargetLibraryInfo *TLibInfo, ArrayRef<const Value *> Scalars) const {
 
   // vXi8 multiplications are always promoted to vXi16.
   // Sub-128-bit types can be extended/packed more efficiently.
@@ -1925,7 +1925,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
 
   // Fallback to the default implementation.
   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                       Args, CtxI, Scalars);
+                                       Args, CtxI, TLibInfo, Scalars);
 }
 
 InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 6c29c53ac448b2..9ddd12f8ab4fbe 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -71,6 +71,7 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
       TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
       ArrayRef<const Value *> Args = {}, const Instruction *CtxI = nullptr,
+      const TargetLibraryInfo *TLibInfo = nullptr,
       ArrayRef<const Value *> Scalars = {}) const override;
   InstructionCost
   getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,

>From f473a9a9e6aded4f67f2b25872b48410eea7eff3 Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Tue, 17 Nov 2026 10:29:33 -0600
Subject: [PATCH 3/4] [TTI] Pass the scalars in the scalarized
 getArithmeticInstrCost

---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index bf532f5e87d447..3cae4102bc97f1 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1145,7 +1145,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
       InstructionCost Cost = thisT()->getArithmeticInstrCost(
           Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info, Args,
-          CtxI, TLibInfo);
+          CtxI, TLibInfo, Scalars);
       // Return the cost of multiple scalar invocation plus the cost of
       // inserting and extracting the values.
       SmallVector<Type *> Tys(Args.size(), Ty);

>From 866d7c9202167905e06e65761e1e32058e371ddd Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Fri, 20 Nov 2026 02:14:24 -0600
Subject: [PATCH 4/4] [SLP][AMDGPU] Apply reviewer suggestions

---
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      | 37 +++++++++++--------
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 24 ++++++++----
 .../AMDGPU/fmul-fusion-copyable-lane.ll       | 23 ++++++++----
 3 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 729cf69da1ff99..eb1f5113e5fba6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -543,14 +543,17 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
 
   // A vector operation has a property of its context instruction only if all
   // lanes of the same opcode have it.
-  auto AllLanes = [&](function_ref<bool(const Instruction *)> Pred) {
-    if (!CtxI)
+  auto AllLanes = [&](function_ref<bool(const Instruction *)> Pred,
+                      bool SkipOtherOpcodes) {
+    if (!CtxI || CtxI->getOpcode() != Opcode || !Pred(CtxI))
       return false;
-    if (Scalars.empty())
-      return Pred(CtxI);
     return all_of(Scalars, [&](const Value *V) {
+      if (!V || isa<PoisonValue>(V))
+        return true;
       const auto *I = dyn_cast<Instruction>(V);
-      return !I || I->getOpcode() != CtxI->getOpcode() || Pred(I);
+      if (!I || I->getOpcode() != Opcode)
+        return SkipOtherOpcodes;
+      return I == CtxI || Pred(I);
     });
   };
 
@@ -604,15 +607,18 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     if (CtxI &&
         (CtxI->getType() == Ty ||
          getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1) &&
-        AllLanes([&](const Instruction *FMul) {
-          if (!FMul->hasOneUse())
-            return false;
-          const auto *FAddSub = dyn_cast<BinaryOperator>(*FMul->user_begin());
-          return FAddSub &&
-                 (FAddSub->getOpcode() == Instruction::FAdd ||
-                  FAddSub->getOpcode() == Instruction::FSub) &&
-                 isFusedFMul(*TLI, Ty, FMul, FAddSub);
-        }))
+        AllLanes(
+            [&](const Instruction *FMul) {
+              if (!FMul->hasOneUse())
+                return false;
+              const auto *FAddSub =
+                  dyn_cast<BinaryOperator>(*FMul->user_begin());
+              return FAddSub &&
+                     (FAddSub->getOpcode() == Instruction::FAdd ||
+                      FAddSub->getOpcode() == Instruction::FSub) &&
+                     isFusedFMul(*TLI, Ty, FMul, FAddSub);
+            },
+            /*SkipOtherOpcodes=*/false))
       return TargetTransformInfo::TCC_Free;
     [[fallthrough]];
   case ISD::FADD:
@@ -667,7 +673,8 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     }
 
     if (SLT == MVT::f32 &&
-        AllLanes([](const Instruction *I) { return I->hasApproxFunc(); })) {
+        AllLanes([](const Instruction *I) { return I->hasApproxFunc(); },
+                 /*SkipOtherOpcodes=*/true)) {
       // Fast unsafe fdiv lowering:
       // f32 rcp
       // f32 fmul
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e08589d091b79..fc9bf07668b777 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -10357,9 +10357,9 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
         Ops1.empty() ? TTI::OperandValueInfo() : TTI::getOperandInfo(Ops1);
     InstructionCost OriginalVecOpsCost =
         TTI->getArithmeticInstrCost(Opcode0, VecTy, CostKind, Op0Info, Op1Info,
-                                    {}, LocalState.getMainOp()) +
+                                    {}, LocalState.getMainOp(), TLI, VL) +
         TTI->getArithmeticInstrCost(Opcode1, VecTy, CostKind, Op0Info, Op1Info,
-                                    {}, LocalState.getAltOp());
+                                    {}, LocalState.getAltOp(), TLI, VL);
     SmallVector<int> OriginalMask(VL.size(), PoisonMaskElem);
     for (unsigned Idx : seq<unsigned>(VL.size())) {
       if (isa<PoisonValue>(VL[Idx]))
@@ -10371,9 +10371,11 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
                                             CostKind, OriginalMask);
     InstructionCost NewVecOpsCost =
         TTI->getArithmeticInstrCost(Opcode0, Op1VecTy, CostKind, Op0Info,
-                                    Op1Info, {}, LocalState.getMainOp()) +
+                                    Op1Info, {}, LocalState.getMainOp(), TLI,
+                                    Op1) +
         TTI->getArithmeticInstrCost(Opcode1, Op2VecTy, CostKind, Op0Info,
-                                    Op1Info, {}, LocalState.getAltOp());
+                                    Op1Info, {}, LocalState.getAltOp(), TLI,
+                                    Op2);
     InstructionCost NewCost =
         NewVecOpsCost + InsertCost +
         (!VectorizableTree.empty() && getRootNode().hasState() &&
@@ -11481,7 +11483,7 @@ class InstructionsCompatibilityAnalysis {
             MainOpcode, VecTy, Kind,
             TTI::commonOperandInfo(Operands[0][0], Operands[0][1]),
             TTI::commonOperandInfo(Operands[1][0], Operands[1][1]), {},
-            S.getMainOp(), &TLI);
+            S.getMainOp(), &TLI, VL);
         break;
       default:
         // Calls (min/max, fmuladd) return above before reaching this switch.
@@ -18462,11 +18464,17 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
           for (unsigned Idx : seq<unsigned>(1, E->getNumOperands())) {
             TTI::OperandValueInfo ColInfo =
                 TTI::getOperandInfo(E->getOperand(Idx));
-            if (!RunningInfo.isConstant() || !ColInfo.isConstant())
+            if (!RunningInfo.isConstant() || !ColInfo.isConstant()) {
+              bool IsFinal = Idx + 1 == E->getNumOperands();
               Cost += TTIRef.getArithmeticInstrCost(
                   Opcode, VecTy, CostKind, RunningInfo, ColInfo, {},
-                  Opcode == E->getOpcode() ? E->getMainOp() : E->getAltOp(),
-                  TLI);
+                  IsFinal ? (Opcode == E->getOpcode() ? E->getMainOp()
+                                                      : E->getAltOp())
+                          : nullptr,
+                  TLI,
+                  IsFinal ? ArrayRef<const Value *>(E->Scalars)
+                          : ArrayRef<const Value *>{});
+            }
             TTI::OperandValueKind Kind = TTI::OK_AnyValue;
             if (RunningInfo.isConstant() && ColInfo.isConstant())
               Kind = RunningInfo.Kind == TTI::OK_UniformConstantValue &&
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-copyable-lane.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-copyable-lane.ll
index c1c4593e00ccfc..5df837cd877265 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-copyable-lane.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-fusion-copyable-lane.ll
@@ -8,15 +8,24 @@
 define double @copyable_lane(ptr %a, ptr %b) {
 ; CHECK-LABEL: define double @copyable_lane(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT:    [[B0:%.*]] = load double, ptr [[B]], align 8
+; CHECK-NEXT:    [[AP1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT:    [[BP1:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, ptr [[AP1]], align 8
+; CHECK-NEXT:    [[B1:%.*]] = load double, ptr [[BP1]], align 8
+; CHECK-NEXT:    [[AP2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[BP2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 2
+; CHECK-NEXT:    [[A2:%.*]] = load double, ptr [[AP2]], align 8
 ; CHECK-NEXT:    [[B2:%.*]] = load double, ptr [[BP2]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[B]], align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x double>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[B2]], i64 2
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP6:%.*]] = fmul reassoc contract <4 x double> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call contract double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP6]])
+; CHECK-NEXT:    [[AP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[A3:%.*]] = load double, ptr [[AP3]], align 8
+; CHECK-NEXT:    [[M0:%.*]] = fmul reassoc contract double [[A0]], [[B0]]
+; CHECK-NEXT:    [[M1:%.*]] = fmul reassoc contract double [[A1]], [[B1]]
+; CHECK-NEXT:    [[M2:%.*]] = fmul reassoc contract double [[A2]], [[B2]]
+; CHECK-NEXT:    [[S1:%.*]] = fadd reassoc contract double [[M0]], [[M1]]
+; CHECK-NEXT:    [[S2:%.*]] = fadd reassoc contract double [[S1]], [[M2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd reassoc contract double [[S2]], [[A3]]
 ; CHECK-NEXT:    ret double [[TMP7]]
 ;
   %a0 = load double, ptr %a



More information about the llvm-branch-commits mailing list