[llvm] [SLP] More accurately cost RISCV scalar splats (PR #213104)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 01:15:02 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/213104
>From d239369ea0ed2be287a687956854dd77021ec79d Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 29 Jul 2026 17:55:18 -0700
Subject: [PATCH 1/9] [SLP] Add tests for overcosted scalar splats for RISCV
---
.../RISCV/folded-broadcast-cost.ll | 58 +++++++++++++++++++
1 file changed, 58 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
new file mode 100644
index 0000000000000..7907954d92afe
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v -S -disable-output -debug-only=SLP 2>&1 | FileCheck %s
+
+; CHECK: SLP: Adding cost 2 for bundle Idx: 2, n=4 [i32 %s, ..].
+define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
+ %x0 = load i32, ptr %p
+ %p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
+ %x1 = load i32, ptr %p1
+ %p2 = getelementptr inbounds nuw i32, ptr %p, i64 2
+ %x2 = load i32, ptr %p2
+ %p3 = getelementptr inbounds nuw i32, ptr %p, i64 3
+ %x3 = load i32, ptr %p3
+ %m0 = add i32 %s, %x0
+ %m1 = add i32 %s, %x1
+ %m2 = add i32 %s, %x2
+ %m3 = add i32 %s, %x3
+ store i32 %m0, ptr %q
+ %q1 = getelementptr i32, ptr %q, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %q, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %q, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
+
+; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 4, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 2 for bundle Idx: 6, n=4 [i32 %s, ..].
+define void @fmul_splat_nested(i32 %s, ptr %p, ptr %q) {
+ %x0 = load i32, ptr %p
+ %p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
+ %x1 = load i32, ptr %p1
+ %p2 = getelementptr inbounds nuw i32, ptr %p, i64 2
+ %x2 = load i32, ptr %p2
+ %p3 = getelementptr inbounds nuw i32, ptr %p, i64 3
+ %x3 = load i32, ptr %p3
+ %b0 = add i32 %s, %x0
+ %b1 = add i32 %s, %x1
+ %b2 = add i32 %s, %x2
+ %b3 = add i32 %s, %x3
+ %a0 = add i32 %s, %b0
+ %a1 = add i32 %s, %b1
+ %a2 = add i32 %s, %b2
+ %a3 = add i32 %s, %b3
+ %m0 = add i32 %s, %a0
+ %m1 = add i32 %s, %a1
+ %m2 = add i32 %s, %a2
+ %m3 = add i32 %s, %a3
+ store i32 %m0, ptr %q
+ %q1 = getelementptr i32, ptr %q, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %q, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %q, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
>From 95ce3d7e3f8f0b2597763feb3683247be66397a2 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 29 Jul 2026 11:07:25 -0700
Subject: [PATCH 2/9] [SLP] More accurately cost RISCV scalar splats
No cost when they are operands to vfoo.vx instructions.
Fixes #212413.
---
.../llvm/Analysis/TargetTransformInfo.h | 8 ++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 14 +++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 9 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 17 ++++++
.../Target/RISCV/RISCVTargetTransformInfo.h | 9 ++-
.../Transforms/Vectorize/SLPVectorizer.cpp | 59 +++++++++++++++----
.../SLPVectorizer/SLPCostAnalysis.cpp | 9 +--
.../Vectorize/SLPVectorizer/SLPCostAnalysis.h | 17 +++---
.../RISCV/check-node-without-vector-user.ll | 42 ++++---------
.../RISCV/folded-broadcast-cost.ll | 4 +-
.../remark-zext-incoming-for-neg-icmp.ll | 2 +-
.../RISCV/reordered-buildvector-scalars.ll | 8 +--
12 files changed, 135 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..e7019705764a6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,6 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
+ SplatOp, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
@@ -1518,6 +1519,10 @@ class TargetTransformInfo {
LLVM_ABI static OperandValueInfo commonOperandInfo(const Value *X,
const Value *Y);
+ /// Return true if a vector instruction with opcode \p Opcode can lower to a
+ /// target instruction able to splat operand \p Operand.
+ LLVM_ABI bool canSplatOperand(unsigned Opcode, int Operand) const;
+
/// This is an approximation of reciprocal throughput of a math/logic op.
/// A higher cost indicates less expected throughput.
/// From Agner Fog's guides, reciprocal throughput is "the average number of
@@ -1568,7 +1573,8 @@ class TargetTransformInfo {
ArrayRef<int> Mask = {},
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, int Index = 0,
VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// Represents a hint about the context in which a cast is used.
///
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 25574a2d2c0e5..2d41a8db947d1 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -784,6 +784,16 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 1;
}
+ virtual InstructionCost
+ getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args,
+ const Instruction *CxtI, TTI::VectorInstrContext VIC) const {
+ (void)VIC;
+ return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp,
+ Args, CxtI);
+ }
+
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
@@ -1143,6 +1153,10 @@ class LLVM_ABI TargetTransformInfoImplBase {
return true;
}
+ virtual bool canSplatOperand(unsigned Opcode, int Operand) const {
+ return false;
+ }
+
virtual bool isElementTypeLegalForScalableVector(Type *Ty) const {
return true;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 595c2d10dd118..95b338946bc70 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1009,6 +1009,10 @@ TargetTransformInfo::commonOperandInfo(const Value *X, const Value *Y) {
return OpInfoX.mergeWith(getOperandInfo(Y));
}
+bool TargetTransformInfo::canSplatOperand(unsigned Opcode, int Operand) const {
+ return TTIImpl->canSplatOperand(Opcode, Operand);
+}
+
InstructionCost TargetTransformInfo::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
OperandValueInfo Op1Info, OperandValueInfo Op2Info,
@@ -1046,14 +1050,15 @@ InstructionCost TargetTransformInfo::getAltInstrCost(
InstructionCost TargetTransformInfo::getShuffleCost(
ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef<int> Mask,
TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args, const Instruction *CxtI) const {
+ ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
"Expected the same scalar types");
InstructionCost Cost = TTIImpl->getShuffleCost(
- Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI);
+ Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index ed49f8b7a201a..af0eaf3dd5ea0 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -999,6 +999,17 @@ RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
SubTp);
}
+InstructionCost RISCVTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
+ if (VIC == TTI::VectorInstrContext::SplatOp)
+ return TTI::TCC_Free;
+ return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
+ CxtI);
+}
+
static unsigned isM1OrSmaller(MVT VT) {
RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
@@ -2561,6 +2572,12 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
VIC);
+ // Scalar splat operand can be folded for vector ops that support splatting
+ // the scalar operand, so the explicit insertelement is free in this context.
+ if (Opcode == Instruction::InsertElement &&
+ VIC == TTI::VectorInstrContext::SplatOp)
+ return TTI::TCC_Free;
+
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 40c6204ee0380..24c0311bf1c2c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -180,6 +180,13 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
+ InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
+ VectorType *SrcTy, ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp,
+ ArrayRef<const Value *> Args,
+ const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
@@ -545,7 +552,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool canSplatOperand(Instruction *I, int Operand) const;
/// Return true if a vector instruction will lower to a target instruction
/// able to splat the given operand.
- bool canSplatOperand(unsigned Opcode, int Operand) const;
+ bool canSplatOperand(unsigned Opcode, int Operand) const override;
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a26b6ae4e52d..cf174910485d1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6441,7 +6441,8 @@ static InstructionCost getScalarizationOverhead(
static InstructionCost getVectorInstrCost(
const TargetTransformInfo &TTI, Type *ScalarTy, unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) {
if (Opcode == Instruction::ExtractElement) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "Only supported by REVEC.");
@@ -6452,7 +6453,7 @@ static InstructionCost getVectorInstrCost(
}
}
return TTI.getVectorInstrCost(Opcode, Val, CostKind, Index, Scalar,
- ScalarUserAndIdx);
+ ScalarUserAndIdx, VIC);
}
/// This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst
@@ -13659,6 +13660,7 @@ class BaseShuffleAnalysis {
Mask.swap(NewMask);
}
+protected:
/// Looks through shuffles trying to reduce final number of shuffles in the
/// code. The function looks through the previously emitted shuffle
/// instructions and properly mark indices in mask as undef.
@@ -15115,6 +15117,7 @@ void BoUpSLP::transformNodes() {
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
+private:
bool IsFinalized = false;
SmallVector<int> CommonMask;
SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
@@ -15142,6 +15145,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
/// May help better estimate the cost if same nodes must be permuted + allows
/// to move most of the long shuffles cost estimation to TTI.
bool SameNodesEstimated = true;
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None;
static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
if (Ty->getScalarType()->isPointerTy()) {
@@ -15182,7 +15186,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
CostKind, std::distance(VL.begin(), It),
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It,
+ ContextHint);
}
SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
@@ -15191,11 +15196,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
});
InstructionCost InsertCost =
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It, ContextHint);
return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast,
VecTy, ShuffleMask, CostKind,
/*Index=*/0, /*SubTp=*/nullptr,
- /*Args=*/*It);
+ /*Args=*/*It, ContextHint);
}
return GatherCost +
(all_of(Gathers, IsaPred<UndefValue>)
@@ -15401,6 +15406,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
class ShuffleCostBuilder {
const TargetTransformInfo &TTI;
+ const TTI::VectorInstrContext ContextHint;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
int Index = -1;
@@ -15412,7 +15418,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
public:
- ShuffleCostBuilder(const TargetTransformInfo &TTI) : TTI(TTI) {}
+ ShuffleCostBuilder(const TargetTransformInfo &TTI,
+ TTI::VectorInstrContext ContextHint)
+ : TTI(TTI), ContextHint(ContextHint) {}
~ShuffleCostBuilder() = default;
InstructionCost createShuffleVector(Value *V1, Value *,
ArrayRef<int> Mask) const {
@@ -15431,9 +15439,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return getShuffleCost(
- TTI, TTI::SK_PermuteSingleSrc, cast<VectorType>(V1->getType()), Mask,
- TTI::TCK_RecipThroughput, /*Index=*/0, /*SubTp=*/nullptr, VL);
+ return getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(V1->getType()), Mask,
+ TTI::TCK_RecipThroughput, /*Index=*/0,
+ /*SubTp=*/nullptr, VL, ContextHint);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
@@ -15449,7 +15458,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
const PointerUnion<Value *, const TreeEntry *> &P2,
ArrayRef<int> Mask, ArrayRef<Value *> VL = {}) {
- ShuffleCostBuilder Builder(TTI);
+ ShuffleCostBuilder Builder(TTI, ContextHint);
SmallVector<int> CommonMask(Mask);
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
unsigned CommonVF = Mask.size();
@@ -15805,6 +15814,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
+ void addContextHint(TTI::VectorInstrContext Ctx) { ContextHint = Ctx; }
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
@@ -15957,6 +15967,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
getAllOnesValue(*R.DL, ScalarTy->getScalarType()));
}
InstructionCost createFreeze(InstructionCost Cost) { return Cost; }
+
/// Finalize emission of the shuffles.
InstructionCost finalize(
ArrayRef<int> ExtMask,
@@ -21850,6 +21861,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
CommonMask.clear();
InVectors.clear();
}
+ void addContextHint(TTI::VectorInstrContext Ctx) { (void)Ctx; }
/// Adds 2 input vectors (in form of tree entries) and the mask for their
/// shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
@@ -22633,6 +22645,33 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
+ // Backends may have a fast path for splatting scalar operands (i.e. rather
+ // than generating the splat vector, the vector instruction may be able to
+ // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
+ // hint to the TTI when costing the insert/shuffle sequence in such cases.
+ if (all_of(ReuseMask, [](const int Idx) { return Idx == 0; })) {
+ Value *SplatVal = E->Scalars.front();
+ if (!isa<VectorType>(SplatVal->getType()) &&
+ !isa<ExtractElementInst>(SplatVal)) {
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() &&
+ "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI->canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOp);
+ }
+ }
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
ShuffleBuilder.add(BV, ReuseMask);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
index 6e20ee9f0b765..9c60e54f60907 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
@@ -26,14 +26,15 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind,
int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args) {
+ ArrayRef<const Value *> Args,
+ TTI::VectorInstrContext VIC) {
VectorType *DstTy = Tp;
if (!Mask.empty())
DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size());
if (Kind != TTI::SK_PermuteTwoSrc)
return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
+ Args, /*CxtI=*/nullptr, VIC);
int NumSrcElts = Tp->getElementCount().getKnownMinValue();
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
@@ -43,8 +44,8 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, Mask,
TTI::TCK_RecipThroughput, Index, Tp);
}
- return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
+ return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp, Args,
+ /*CxtI=*/nullptr, VIC);
}
std::pair<InstructionCost, InstructionCost>
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
index 333675d483a30..accd8b4f348b3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
@@ -33,13 +33,16 @@ namespace llvm::slpvectorizer {
/// Returns the cost of the shuffle instructions with the given \p Kind, vector
/// type \p Tp and optional \p Mask. Adds SLP-specific cost estimation for
/// insert subvector pattern.
-InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
- TargetTransformInfo::ShuffleKind Kind,
- VectorType *Tp, ArrayRef<int> Mask = {},
- TargetTransformInfo::TargetCostKind CostKind =
- TargetTransformInfo::TCK_RecipThroughput,
- int Index = 0, VectorType *SubTp = nullptr,
- ArrayRef<const Value *> Args = {});
+InstructionCost
+getShuffleCost(const TargetTransformInfo &TTI,
+ TargetTransformInfo::ShuffleKind Kind, VectorType *Tp,
+ ArrayRef<int> Mask = {},
+ TargetTransformInfo::TargetCostKind CostKind =
+ TargetTransformInfo::TCK_RecipThroughput,
+ int Index = 0, VectorType *SubTp = nullptr,
+ ArrayRef<const Value *> Args = {},
+ TargetTransformInfo::VectorInstrContext VIC =
+ TargetTransformInfo::VectorInstrContext::None);
/// Calculate the scalar and the vector costs from vectorizing set of GEPs.
std::pair<InstructionCost, InstructionCost>
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
index 8f738c3978029..b5dbe1b089df8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
@@ -6,37 +6,17 @@
define void @test(i64 %0, ptr %1) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr @r, align 1
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i8 [[TMP3]] to i1
-; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr @r, i64 [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule i8 [[TMP3]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = sext i1 [[TMP8]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -8049), align 1
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i8 [[TMP10]] to i1
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr @r, i64 [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ule i8 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = sext i1 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP9]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -16098), align 1
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i8 [[TMP18]] to i1
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr @r, i64 [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP23:%.*]] = icmp ule i8 [[TMP18]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = sext i1 [[TMP23]] to i32
-; CHECK-NEXT: [[TMP25:%.*]] = add i32 [[TMP17]], [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -24147), align 1
-; CHECK-NEXT: [[TMP27:%.*]] = trunc i8 [[TMP26]] to i1
-; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr @r, i64 [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[TMP29]], align 1
-; CHECK-NEXT: [[TMP31:%.*]] = icmp ule i8 [[TMP26]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = sext i1 [[TMP31]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP25]], [[TMP32]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr align 1 getelementptr (i8, ptr @r, i64 -24147), i64 8049, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> <ptr @r, ptr @r, ptr @r, ptr @r>, <4 x i64> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP8]], <4 x i1> splat (i1 true), <4 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ule <4 x i8> [[TMP3]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i8>
+; CHECK-NEXT: [[TMP12:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP11]])
+; CHECK-NEXT: [[TMP33:%.*]] = sext i8 [[TMP12]] to i32
; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP1]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
index 7907954d92afe..4c771d76a3f7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v -S -disable-output -debug-only=SLP 2>&1 | FileCheck %s
-; CHECK: SLP: Adding cost 2 for bundle Idx: 2, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
%x0 = load i32, ptr %p
%p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
@@ -26,7 +26,7 @@ define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
; CHECK: SLP: Adding cost 0 for bundle Idx: 4, n=4 [i32 %s, ..].
-; CHECK: SLP: Adding cost 2 for bundle Idx: 6, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 6, n=4 [i32 %s, ..].
define void @fmul_splat_nested(i32 %s, ptr %p, ptr %q) {
%x0 = load i32, ptr %p
%p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
index 5ef40a3f6d2f7..8c640da2fa718 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: test
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost '
-; YAML-NEXT: - Cost: '-10'
+; YAML-NEXT: - Cost: '-12'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '8'
; YAML-NEXT:...
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
index 9a9fcd1b259c3..6501394690d97 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
@@ -54,10 +54,10 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-SAME: i32 [[TMP0:%.*]], i32 [[ADD111_I_I:%.*]], <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT:%.*]], <4 x i32> [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; THRESH-NEXT: [[ENTRY:.*:]]
; THRESH-NEXT: [[MUL1445_I:%.*]] = shl i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2235_I16:%.*]] = or i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
; THRESH-NEXT: [[TMP31:%.*]] = shufflevector <2 x i32> [[TMP30]], <2 x i32> poison, <2 x i32> zeroinitializer
+; THRESH-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP31]], <i32 1, i32 0>
+; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP32:%.*]] = add <2 x i32> [[TMP31]], <i32 0, i32 1>
; THRESH-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP32]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
; THRESH-NEXT: [[TMP21:%.*]] = ashr <4 x i32> [[TMP5]], <i32 1, i32 0, i32 0, i32 1>
@@ -71,8 +71,8 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 poison, i32 poison, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <3 x i32> [[TMP4]], <3 x i32> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 0, i32 poison>
; THRESH-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 poison, i32 10, i32 3, i32 poison, i32 poison, i32 14, i32 poison>
-; THRESH-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP0]], i64 4
-; THRESH-NEXT: [[TMP36:%.*]] = insertelement <8 x i32> [[TMP35]], i32 [[ADD2235_I16]], i64 1
+; THRESH-NEXT: [[TMP35:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; THRESH-NEXT: [[TMP36:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP35]], <8 x i32> <i32 0, i32 8, i32 2, i32 3, i32 11, i32 5, i32 6, i32 7>
; THRESH-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[MUL1445_I]], i64 5
; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
>From 3a141a10ed1a79965fa118c703d6931faea764a1 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 12:20:36 -0700
Subject: [PATCH 3/9] Remove extra class property's
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cf174910485d1..40851d727ee6c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13660,7 +13660,6 @@ class BaseShuffleAnalysis {
Mask.swap(NewMask);
}
-protected:
/// Looks through shuffles trying to reduce final number of shuffles in the
/// code. The function looks through the previously emitted shuffle
/// instructions and properly mark indices in mask as undef.
@@ -15117,7 +15116,6 @@ void BoUpSLP::transformNodes() {
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
-private:
bool IsFinalized = false;
SmallVector<int> CommonMask;
SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
>From 64400f177f4dd876d33cad9d9a64da8f2a51c6a5 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:05:20 -0700
Subject: [PATCH 4/9] SplatOp -> SplatOpFolded
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index e7019705764a6..f31967e265469 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,7 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
- SplatOp, ///< The value's user supports a scalar splat operand
+ SplatOpFolded, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index af0eaf3dd5ea0..0579d54f9702a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1004,7 +1004,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
TTI::VectorInstrContext VIC) const {
- if (VIC == TTI::VectorInstrContext::SplatOp)
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded)
return TTI::TCC_Free;
return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
CxtI);
@@ -2575,7 +2575,7 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
// Scalar splat operand can be folded for vector ops that support splatting
// the scalar operand, so the explicit insertelement is free in this context.
if (Opcode == Instruction::InsertElement &&
- VIC == TTI::VectorInstrContext::SplatOp)
+ VIC == TTI::VectorInstrContext::SplatOpFolded)
return TTI::TCC_Free;
// Legalize the type.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 40851d727ee6c..119a94febafec 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22667,7 +22667,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
return TTI->canSplatOperand(UserTE->getOpcode(),
TE->UserTreeIndex.EdgeIdx);
}))
- ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOp);
+ ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOpFolded);
}
}
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
>From eb61ec3b9988f592e57fa7124d69f833ff0c494a Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:20:55 -0700
Subject: [PATCH 5/9] Don't overload getShuffleCost
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 21 +++++-----------
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 ++-
.../AArch64/AArch64TargetTransformInfo.cpp | 11 ++++----
.../AArch64/AArch64TargetTransformInfo.h | 4 ++-
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 ++++-----
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 4 ++-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 12 ++++-----
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 4 ++-
.../Hexagon/HexagonTargetTransformInfo.cpp | 11 ++++----
.../Hexagon/HexagonTargetTransformInfo.h | 4 ++-
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 12 ++++-----
.../Target/PowerPC/PPCTargetTransformInfo.h | 4 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 25 ++++++-------------
.../Target/RISCV/RISCVTargetTransformInfo.h | 11 +++-----
.../SystemZ/SystemZTargetTransformInfo.cpp | 11 ++++----
.../SystemZ/SystemZTargetTransformInfo.h | 4 ++-
.../WebAssemblyTargetTransformInfo.cpp | 4 +--
.../WebAssemblyTargetTransformInfo.h | 4 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 12 ++++-----
llvm/lib/Target/X86/X86TargetTransformInfo.h | 4 ++-
20 files changed, 81 insertions(+), 97 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 2d41a8db947d1..da8e9e0176915 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -776,24 +776,15 @@ class LLVM_ABI TargetTransformInfoImplBase {
return InstructionCost::getInvalid();
}
- virtual InstructionCost
- getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const {
+ virtual InstructionCost getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args = {},
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
- virtual InstructionCost
- getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI, TTI::VectorInstrContext VIC) const {
- (void)VIC;
- return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp,
- Args, CxtI);
- }
-
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..6fbe29987d969 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1218,7 +1218,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override {
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
switch (improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp)) {
case TTI::SK_Broadcast:
if (auto *FVT = dyn_cast<FixedVectorType>(SrcTy))
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 341ac1008d7e7..70e6755b23095 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6557,12 +6557,11 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
BinOp, CostKind, FMF);
}
-InstructionCost
-AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost AArch64TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ddca341bad05b..1a20249357324 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -507,7 +507,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a5ee5fcd76031..30f7bb2f0b03c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1334,13 +1334,11 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
}
}
-InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost GCNTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
if (!isa<FixedVectorType>(SrcTy))
return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,
SubTp);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index df7b6d339e6c2..091ae53c15a50 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -250,7 +250,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 7d4f9ae5b023b..66f5b5aef68b2 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1224,13 +1224,11 @@ InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
return NumOps;
}
-InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost ARMTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 19bd6e7e09366..2d339cbf61aba 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -195,7 +195,9 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool preferInLoopReduction(RecurKind Kind, Type *Ty) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 8ff79f7f4f54d..104c762fc2f1d 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -231,12 +231,11 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
-InstructionCost
-HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost HexagonTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
return 1;
}
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 39517f8c64c88..3f80c9d69bbde 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -124,7 +124,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b3a54fd4c1ef4..800485d5cfed1 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -617,13 +617,11 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
return Cost * CostFactor;
}
-InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost PPCTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
InstructionCost CostFactor =
vectorCostAdjustmentFactor(Instruction::ShuffleVector, SrcTy, nullptr);
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index c8ec1960b17dd..d2e116b68b60b 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -115,7 +115,9 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 0579d54f9702a..572407ea387bf 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -673,12 +673,14 @@ InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
return FirstSlideCost + SecondSlideCost + MaskCost;
}
-InstructionCost
-RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost RISCVTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded)
+ return TTI::TCC_Free;
+
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
@@ -999,17 +1001,6 @@ RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
SubTp);
}
-InstructionCost RISCVTTIImpl::getShuffleCost(
- TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
- TTI::VectorInstrContext VIC) const {
- if (VIC == TTI::VectorInstrContext::SplatOpFolded)
- return TTI::TCC_Free;
- return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
- CxtI);
-}
-
static unsigned isM1OrSmaller(MVT VT) {
RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 24c0311bf1c2c..a890d68bdc82e 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -179,14 +179,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
- InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI,
- TTI::VectorInstrContext VIC) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 5353c36fd9b97..1464924a007b5 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -772,12 +772,11 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
Args, CxtI);
}
-InstructionCost
-SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost SystemZTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
if (ST->hasVector()) {
unsigned NumVectors = getNumVectorRegs(SrcTy);
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 06f39e585cd47..2f1ac2277250a 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -118,7 +118,9 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const;
unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const;
unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 7d1136bb9beba..5fb26efaed296 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -302,8 +302,8 @@ InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
InstructionCost WebAssemblyTTIImpl::getShuffleCost(
TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
// Canonicalize the ShuffleKind in case optimizations didn't.
// Otherwise, we might end up with the wrong ShuffleKind to match against.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 221f32609b2de..f63024a03fc4f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -90,7 +90,9 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
using BaseT::getVectorInstrCost;
InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 421a2829cb509..8cc1281ed82c2 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1580,13 +1580,11 @@ X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
return InstructionCost::getInvalid();
}
-InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost X86TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 22171f5469d98..c262db0ec8df4 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -81,7 +81,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
>From cf1c39c9f021b42572f8f5fb9319fdd6b36f0e99 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:38:14 -0700
Subject: [PATCH 6/9] Use isZeroEltSplatMask
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 119a94febafec..df8b47cbd3ffd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22647,7 +22647,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// than generating the splat vector, the vector instruction may be able to
// take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
// hint to the TTI when costing the insert/shuffle sequence in such cases.
- if (all_of(ReuseMask, [](const int Idx) { return Idx == 0; })) {
+ if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
Value *SplatVal = E->Scalars.front();
if (!isa<VectorType>(SplatVal->getType()) &&
!isa<ExtractElementInst>(SplatVal)) {
>From 12124ad792245cdf2190ac4a59e622bdeaf6e208 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 19:06:51 -0700
Subject: [PATCH 7/9] Add checkForFoldingSplat to ShuffleCostEstimator
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 61 ++++++++++---------
1 file changed, 32 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index df8b47cbd3ffd..d6688b71e55be 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15812,7 +15812,35 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
- void addContextHint(TTI::VectorInstrContext Ctx) { ContextHint = Ctx; }
+ void checkForFoldingSplat(const TreeEntry *E, ArrayRef<int> ReuseMask) {
+ // Backends may have a fast path for splatting scalar operands (i.e. rather
+ // than generating the splat vector, the vector instruction may be able to
+ // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
+ // hint to the TTI when costing the insert/shuffle sequence in such cases.
+ if (!ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size()))
+ return;
+ Value *SplatVal = E->Scalars.front();
+ if (isa<VectorType>(SplatVal->getType()) ||
+ !isa<ExtractElementInst>(SplatVal))
+ return;
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : R.VectorizableTree) {
+ if (R.DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() && "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI.canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ ContextHint = TTI::VectorInstrContext::SplatOpFolded;
+ }
+
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
@@ -21859,7 +21887,6 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
CommonMask.clear();
InVectors.clear();
}
- void addContextHint(TTI::VectorInstrContext Ctx) { (void)Ctx; }
/// Adds 2 input vectors (in form of tree entries) and the mask for their
/// shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
@@ -22643,33 +22670,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- // Backends may have a fast path for splatting scalar operands (i.e. rather
- // than generating the splat vector, the vector instruction may be able to
- // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
- // hint to the TTI when costing the insert/shuffle sequence in such cases.
- if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
- Value *SplatVal = E->Scalars.front();
- if (!isa<VectorType>(SplatVal->getType()) &&
- !isa<ExtractElementInst>(SplatVal)) {
- SmallVector<TreeEntry *> MatchingTEs;
- for (const auto &TE : VectorizableTree) {
- if (DeletedNodes.contains(TE.get()))
- continue;
- if (TE->isGather() && E->isSame(TE->Scalars))
- MatchingTEs.emplace_back(TE.get());
- }
- assert(MatchingTEs.size() &&
- "Ought to at least match with current entry");
- if (all_of(MatchingTEs, [this](auto *TE) {
- auto *UserTE = TE->UserTreeIndex.UserTE;
- if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
- return false;
- return TTI->canSplatOperand(UserTE->getOpcode(),
- TE->UserTreeIndex.EdgeIdx);
- }))
- ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOpFolded);
- }
- }
+ if constexpr (std::is_same_v<BVTy, ShuffleCostEstimator>)
+ ShuffleBuilder.checkForFoldingSplat(E, ReuseMask);
+
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
ShuffleBuilder.add(BV, ReuseMask);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
>From 27c02b92b24bc120370c7a68e9ea70b7d1155b7c Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 01:14:04 -0700
Subject: [PATCH 8/9] Missed adding this before pushing
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d6688b71e55be..b04ea93df8656 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15821,7 +15821,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return;
Value *SplatVal = E->Scalars.front();
if (isa<VectorType>(SplatVal->getType()) ||
- !isa<ExtractElementInst>(SplatVal))
+ isa<ExtractElementInst>(SplatVal))
return;
SmallVector<TreeEntry *> MatchingTEs;
for (const auto &TE : R.VectorizableTree) {
>From 05015a83ac13fc55422c46fac84c785214ec9892 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 01:14:44 -0700
Subject: [PATCH 9/9] Format
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index f31967e265469..dd241ad5f0190 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -190,8 +190,8 @@ enum class VectorInstrContext : uint8_t {
None, ///< The instruction is not folded.
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
- BinaryOp, ///< One of the operands is a binary op.
- SplatOpFolded, ///< The value's user supports a scalar splat operand
+ BinaryOp, ///< One of the operands is a binary op.
+ SplatOpFolded, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
More information about the llvm-commits
mailing list