[llvm] [SLP] More accurately cost RISCV scalar splats (PR #213104)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 15:07:13 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/213104
>From d239369ea0ed2be287a687956854dd77021ec79d Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 29 Jul 2026 17:55:18 -0700
Subject: [PATCH 01/19] [SLP] Add tests for overcosted scalar splats for RISCV
---
.../RISCV/folded-broadcast-cost.ll | 58 +++++++++++++++++++
1 file changed, 58 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
new file mode 100644
index 0000000000000..7907954d92afe
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v -S -disable-output -debug-only=SLP 2>&1 | FileCheck %s
+
+; CHECK: SLP: Adding cost 2 for bundle Idx: 2, n=4 [i32 %s, ..].
+define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
+ %x0 = load i32, ptr %p
+ %p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
+ %x1 = load i32, ptr %p1
+ %p2 = getelementptr inbounds nuw i32, ptr %p, i64 2
+ %x2 = load i32, ptr %p2
+ %p3 = getelementptr inbounds nuw i32, ptr %p, i64 3
+ %x3 = load i32, ptr %p3
+ %m0 = add i32 %s, %x0
+ %m1 = add i32 %s, %x1
+ %m2 = add i32 %s, %x2
+ %m3 = add i32 %s, %x3
+ store i32 %m0, ptr %q
+ %q1 = getelementptr i32, ptr %q, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %q, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %q, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
+
+; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 4, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 2 for bundle Idx: 6, n=4 [i32 %s, ..].
+define void @fmul_splat_nested(i32 %s, ptr %p, ptr %q) {
+ %x0 = load i32, ptr %p
+ %p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
+ %x1 = load i32, ptr %p1
+ %p2 = getelementptr inbounds nuw i32, ptr %p, i64 2
+ %x2 = load i32, ptr %p2
+ %p3 = getelementptr inbounds nuw i32, ptr %p, i64 3
+ %x3 = load i32, ptr %p3
+ %b0 = add i32 %s, %x0
+ %b1 = add i32 %s, %x1
+ %b2 = add i32 %s, %x2
+ %b3 = add i32 %s, %x3
+ %a0 = add i32 %s, %b0
+ %a1 = add i32 %s, %b1
+ %a2 = add i32 %s, %b2
+ %a3 = add i32 %s, %b3
+ %m0 = add i32 %s, %a0
+ %m1 = add i32 %s, %a1
+ %m2 = add i32 %s, %a2
+ %m3 = add i32 %s, %a3
+ store i32 %m0, ptr %q
+ %q1 = getelementptr i32, ptr %q, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %q, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %q, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
>From 95ce3d7e3f8f0b2597763feb3683247be66397a2 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 29 Jul 2026 11:07:25 -0700
Subject: [PATCH 02/19] [SLP] More accurately cost RISCV scalar splats
No cost when they are operands to vfoo.vx instructions.
Fixes #212413.
---
.../llvm/Analysis/TargetTransformInfo.h | 8 ++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 14 +++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 9 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 17 ++++++
.../Target/RISCV/RISCVTargetTransformInfo.h | 9 ++-
.../Transforms/Vectorize/SLPVectorizer.cpp | 59 +++++++++++++++----
.../SLPVectorizer/SLPCostAnalysis.cpp | 9 +--
.../Vectorize/SLPVectorizer/SLPCostAnalysis.h | 17 +++---
.../RISCV/check-node-without-vector-user.ll | 42 ++++---------
.../RISCV/folded-broadcast-cost.ll | 4 +-
.../remark-zext-incoming-for-neg-icmp.ll | 2 +-
.../RISCV/reordered-buildvector-scalars.ll | 8 +--
12 files changed, 135 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..e7019705764a6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,6 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
+ SplatOp, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
@@ -1518,6 +1519,10 @@ class TargetTransformInfo {
LLVM_ABI static OperandValueInfo commonOperandInfo(const Value *X,
const Value *Y);
+ /// Return true if a vector instruction with opcode \p Opcode can lower to a
+ /// target instruction able to splat operand \p Operand.
+ LLVM_ABI bool canSplatOperand(unsigned Opcode, int Operand) const;
+
/// This is an approximation of reciprocal throughput of a math/logic op.
/// A higher cost indicates less expected throughput.
/// From Agner Fog's guides, reciprocal throughput is "the average number of
@@ -1568,7 +1573,8 @@ class TargetTransformInfo {
ArrayRef<int> Mask = {},
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput, int Index = 0,
VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// Represents a hint about the context in which a cast is used.
///
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 25574a2d2c0e5..2d41a8db947d1 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -784,6 +784,16 @@ class LLVM_ABI TargetTransformInfoImplBase {
return 1;
}
+ virtual InstructionCost
+ getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args,
+ const Instruction *CxtI, TTI::VectorInstrContext VIC) const {
+ (void)VIC;
+ return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp,
+ Args, CxtI);
+ }
+
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
@@ -1143,6 +1153,10 @@ class LLVM_ABI TargetTransformInfoImplBase {
return true;
}
+ virtual bool canSplatOperand(unsigned Opcode, int Operand) const {
+ return false;
+ }
+
virtual bool isElementTypeLegalForScalableVector(Type *Ty) const {
return true;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 595c2d10dd118..95b338946bc70 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1009,6 +1009,10 @@ TargetTransformInfo::commonOperandInfo(const Value *X, const Value *Y) {
return OpInfoX.mergeWith(getOperandInfo(Y));
}
+bool TargetTransformInfo::canSplatOperand(unsigned Opcode, int Operand) const {
+ return TTIImpl->canSplatOperand(Opcode, Operand);
+}
+
InstructionCost TargetTransformInfo::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
OperandValueInfo Op1Info, OperandValueInfo Op2Info,
@@ -1046,14 +1050,15 @@ InstructionCost TargetTransformInfo::getAltInstrCost(
InstructionCost TargetTransformInfo::getShuffleCost(
ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef<int> Mask,
TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args, const Instruction *CxtI) const {
+ ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
"Expected the same scalar types");
InstructionCost Cost = TTIImpl->getShuffleCost(
- Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI);
+ Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index ed49f8b7a201a..af0eaf3dd5ea0 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -999,6 +999,17 @@ RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
SubTp);
}
+InstructionCost RISCVTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
+ if (VIC == TTI::VectorInstrContext::SplatOp)
+ return TTI::TCC_Free;
+ return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
+ CxtI);
+}
+
static unsigned isM1OrSmaller(MVT VT) {
RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
@@ -2561,6 +2572,12 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
VIC);
+ // Scalar splat operand can be folded for vector ops that support splatting
+ // the scalar operand, so the explicit insertelement is free in this context.
+ if (Opcode == Instruction::InsertElement &&
+ VIC == TTI::VectorInstrContext::SplatOp)
+ return TTI::TCC_Free;
+
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 40c6204ee0380..24c0311bf1c2c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -180,6 +180,13 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
+ InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
+ VectorType *SrcTy, ArrayRef<int> Mask,
+ TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp,
+ ArrayRef<const Value *> Args,
+ const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
@@ -545,7 +552,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool canSplatOperand(Instruction *I, int Operand) const;
/// Return true if a vector instruction will lower to a target instruction
/// able to splat the given operand.
- bool canSplatOperand(unsigned Opcode, int Operand) const;
+ bool canSplatOperand(unsigned Opcode, int Operand) const override;
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a26b6ae4e52d..cf174910485d1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6441,7 +6441,8 @@ static InstructionCost getScalarizationOverhead(
static InstructionCost getVectorInstrCost(
const TargetTransformInfo &TTI, Type *ScalarTy, unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) {
if (Opcode == Instruction::ExtractElement) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "Only supported by REVEC.");
@@ -6452,7 +6453,7 @@ static InstructionCost getVectorInstrCost(
}
}
return TTI.getVectorInstrCost(Opcode, Val, CostKind, Index, Scalar,
- ScalarUserAndIdx);
+ ScalarUserAndIdx, VIC);
}
/// This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst
@@ -13659,6 +13660,7 @@ class BaseShuffleAnalysis {
Mask.swap(NewMask);
}
+protected:
/// Looks through shuffles trying to reduce final number of shuffles in the
/// code. The function looks through the previously emitted shuffle
/// instructions and properly mark indices in mask as undef.
@@ -15115,6 +15117,7 @@ void BoUpSLP::transformNodes() {
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
+private:
bool IsFinalized = false;
SmallVector<int> CommonMask;
SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
@@ -15142,6 +15145,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
/// May help better estimate the cost if same nodes must be permuted + allows
/// to move most of the long shuffles cost estimation to TTI.
bool SameNodesEstimated = true;
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None;
static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
if (Ty->getScalarType()->isPointerTy()) {
@@ -15182,7 +15186,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
CostKind, std::distance(VL.begin(), It),
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It,
+ ContextHint);
}
SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
@@ -15191,11 +15196,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
});
InstructionCost InsertCost =
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It, ContextHint);
return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast,
VecTy, ShuffleMask, CostKind,
/*Index=*/0, /*SubTp=*/nullptr,
- /*Args=*/*It);
+ /*Args=*/*It, ContextHint);
}
return GatherCost +
(all_of(Gathers, IsaPred<UndefValue>)
@@ -15401,6 +15406,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
class ShuffleCostBuilder {
const TargetTransformInfo &TTI;
+ const TTI::VectorInstrContext ContextHint;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
int Index = -1;
@@ -15412,7 +15418,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
public:
- ShuffleCostBuilder(const TargetTransformInfo &TTI) : TTI(TTI) {}
+ ShuffleCostBuilder(const TargetTransformInfo &TTI,
+ TTI::VectorInstrContext ContextHint)
+ : TTI(TTI), ContextHint(ContextHint) {}
~ShuffleCostBuilder() = default;
InstructionCost createShuffleVector(Value *V1, Value *,
ArrayRef<int> Mask) const {
@@ -15431,9 +15439,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return getShuffleCost(
- TTI, TTI::SK_PermuteSingleSrc, cast<VectorType>(V1->getType()), Mask,
- TTI::TCK_RecipThroughput, /*Index=*/0, /*SubTp=*/nullptr, VL);
+ return getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
+ cast<VectorType>(V1->getType()), Mask,
+ TTI::TCK_RecipThroughput, /*Index=*/0,
+ /*SubTp=*/nullptr, VL, ContextHint);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
@@ -15449,7 +15458,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
const PointerUnion<Value *, const TreeEntry *> &P2,
ArrayRef<int> Mask, ArrayRef<Value *> VL = {}) {
- ShuffleCostBuilder Builder(TTI);
+ ShuffleCostBuilder Builder(TTI, ContextHint);
SmallVector<int> CommonMask(Mask);
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
unsigned CommonVF = Mask.size();
@@ -15805,6 +15814,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
+ void addContextHint(TTI::VectorInstrContext Ctx) { ContextHint = Ctx; }
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
@@ -15957,6 +15967,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
getAllOnesValue(*R.DL, ScalarTy->getScalarType()));
}
InstructionCost createFreeze(InstructionCost Cost) { return Cost; }
+
/// Finalize emission of the shuffles.
InstructionCost finalize(
ArrayRef<int> ExtMask,
@@ -21850,6 +21861,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
CommonMask.clear();
InVectors.clear();
}
+ void addContextHint(TTI::VectorInstrContext Ctx) { (void)Ctx; }
/// Adds 2 input vectors (in form of tree entries) and the mask for their
/// shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
@@ -22633,6 +22645,33 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
+ // Backends may have a fast path for splatting scalar operands (i.e. rather
+ // than generating the splat vector, the vector instruction may be able to
+ // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
+ // hint to the TTI when costing the insert/shuffle sequence in such cases.
+ if (all_of(ReuseMask, [](const int Idx) { return Idx == 0; })) {
+ Value *SplatVal = E->Scalars.front();
+ if (!isa<VectorType>(SplatVal->getType()) &&
+ !isa<ExtractElementInst>(SplatVal)) {
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() &&
+ "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI->canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOp);
+ }
+ }
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
ShuffleBuilder.add(BV, ReuseMask);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
index 6e20ee9f0b765..9c60e54f60907 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
@@ -26,14 +26,15 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind,
int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args) {
+ ArrayRef<const Value *> Args,
+ TTI::VectorInstrContext VIC) {
VectorType *DstTy = Tp;
if (!Mask.empty())
DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size());
if (Kind != TTI::SK_PermuteTwoSrc)
return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
+ Args, /*CxtI=*/nullptr, VIC);
int NumSrcElts = Tp->getElementCount().getKnownMinValue();
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
@@ -43,8 +44,8 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, Mask,
TTI::TCK_RecipThroughput, Index, Tp);
}
- return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp,
- Args);
+ return TTI.getShuffleCost(Kind, DstTy, Tp, Mask, CostKind, Index, SubTp, Args,
+ /*CxtI=*/nullptr, VIC);
}
std::pair<InstructionCost, InstructionCost>
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
index 333675d483a30..accd8b4f348b3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
@@ -33,13 +33,16 @@ namespace llvm::slpvectorizer {
/// Returns the cost of the shuffle instructions with the given \p Kind, vector
/// type \p Tp and optional \p Mask. Adds SLP-specific cost estimation for
/// insert subvector pattern.
-InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
- TargetTransformInfo::ShuffleKind Kind,
- VectorType *Tp, ArrayRef<int> Mask = {},
- TargetTransformInfo::TargetCostKind CostKind =
- TargetTransformInfo::TCK_RecipThroughput,
- int Index = 0, VectorType *SubTp = nullptr,
- ArrayRef<const Value *> Args = {});
+InstructionCost
+getShuffleCost(const TargetTransformInfo &TTI,
+ TargetTransformInfo::ShuffleKind Kind, VectorType *Tp,
+ ArrayRef<int> Mask = {},
+ TargetTransformInfo::TargetCostKind CostKind =
+ TargetTransformInfo::TCK_RecipThroughput,
+ int Index = 0, VectorType *SubTp = nullptr,
+ ArrayRef<const Value *> Args = {},
+ TargetTransformInfo::VectorInstrContext VIC =
+ TargetTransformInfo::VectorInstrContext::None);
/// Calculate the scalar and the vector costs from vectorizing set of GEPs.
std::pair<InstructionCost, InstructionCost>
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
index 8f738c3978029..b5dbe1b089df8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
@@ -6,37 +6,17 @@
define void @test(i64 %0, ptr %1) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr @r, align 1
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i8 [[TMP3]] to i1
-; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr @r, i64 [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule i8 [[TMP3]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = sext i1 [[TMP8]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -8049), align 1
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i8 [[TMP10]] to i1
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr @r, i64 [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ule i8 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = sext i1 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP9]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -16098), align 1
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i8 [[TMP18]] to i1
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr @r, i64 [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP23:%.*]] = icmp ule i8 [[TMP18]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = sext i1 [[TMP23]] to i32
-; CHECK-NEXT: [[TMP25:%.*]] = add i32 [[TMP17]], [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -24147), align 1
-; CHECK-NEXT: [[TMP27:%.*]] = trunc i8 [[TMP26]] to i1
-; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr @r, i64 [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[TMP29]], align 1
-; CHECK-NEXT: [[TMP31:%.*]] = icmp ule i8 [[TMP26]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = sext i1 [[TMP31]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP25]], [[TMP32]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr align 1 getelementptr (i8, ptr @r, i64 -24147), i64 8049, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> <ptr @r, ptr @r, ptr @r, ptr @r>, <4 x i64> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP8]], <4 x i1> splat (i1 true), <4 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ule <4 x i8> [[TMP3]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i8>
+; CHECK-NEXT: [[TMP12:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP11]])
+; CHECK-NEXT: [[TMP33:%.*]] = sext i8 [[TMP12]] to i32
; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP1]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
index 7907954d92afe..4c771d76a3f7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v -S -disable-output -debug-only=SLP 2>&1 | FileCheck %s
-; CHECK: SLP: Adding cost 2 for bundle Idx: 2, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
%x0 = load i32, ptr %p
%p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
@@ -26,7 +26,7 @@ define void @fmul_splat(i32 %s, ptr %p, ptr %q) {
; CHECK: SLP: Adding cost 0 for bundle Idx: 2, n=4 [i32 %s, ..].
; CHECK: SLP: Adding cost 0 for bundle Idx: 4, n=4 [i32 %s, ..].
-; CHECK: SLP: Adding cost 2 for bundle Idx: 6, n=4 [i32 %s, ..].
+; CHECK: SLP: Adding cost 0 for bundle Idx: 6, n=4 [i32 %s, ..].
define void @fmul_splat_nested(i32 %s, ptr %p, ptr %q) {
%x0 = load i32, ptr %p
%p1 = getelementptr inbounds nuw i32, ptr %p, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
index 5ef40a3f6d2f7..8c640da2fa718 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: test
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost '
-; YAML-NEXT: - Cost: '-10'
+; YAML-NEXT: - Cost: '-12'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '8'
; YAML-NEXT:...
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
index 9a9fcd1b259c3..6501394690d97 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
@@ -54,10 +54,10 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-SAME: i32 [[TMP0:%.*]], i32 [[ADD111_I_I:%.*]], <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT:%.*]], <4 x i32> [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; THRESH-NEXT: [[ENTRY:.*:]]
; THRESH-NEXT: [[MUL1445_I:%.*]] = shl i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2235_I16:%.*]] = or i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
; THRESH-NEXT: [[TMP31:%.*]] = shufflevector <2 x i32> [[TMP30]], <2 x i32> poison, <2 x i32> zeroinitializer
+; THRESH-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP31]], <i32 1, i32 0>
+; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP32:%.*]] = add <2 x i32> [[TMP31]], <i32 0, i32 1>
; THRESH-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP32]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
; THRESH-NEXT: [[TMP21:%.*]] = ashr <4 x i32> [[TMP5]], <i32 1, i32 0, i32 0, i32 1>
@@ -71,8 +71,8 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 poison, i32 poison, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <3 x i32> [[TMP4]], <3 x i32> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 0, i32 poison>
; THRESH-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 poison, i32 10, i32 3, i32 poison, i32 poison, i32 14, i32 poison>
-; THRESH-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP0]], i64 4
-; THRESH-NEXT: [[TMP36:%.*]] = insertelement <8 x i32> [[TMP35]], i32 [[ADD2235_I16]], i64 1
+; THRESH-NEXT: [[TMP35:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; THRESH-NEXT: [[TMP36:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP35]], <8 x i32> <i32 0, i32 8, i32 2, i32 3, i32 11, i32 5, i32 6, i32 7>
; THRESH-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[MUL1445_I]], i64 5
; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
>From 3a141a10ed1a79965fa118c703d6931faea764a1 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 12:20:36 -0700
Subject: [PATCH 03/19] Remove extra class property's
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cf174910485d1..40851d727ee6c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13660,7 +13660,6 @@ class BaseShuffleAnalysis {
Mask.swap(NewMask);
}
-protected:
/// Looks through shuffles trying to reduce final number of shuffles in the
/// code. The function looks through the previously emitted shuffle
/// instructions and properly mark indices in mask as undef.
@@ -15117,7 +15116,6 @@ void BoUpSLP::transformNodes() {
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
-private:
bool IsFinalized = false;
SmallVector<int> CommonMask;
SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
>From 64400f177f4dd876d33cad9d9a64da8f2a51c6a5 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:05:20 -0700
Subject: [PATCH 04/19] SplatOp -> SplatOpFolded
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index e7019705764a6..f31967e265469 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,7 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
- SplatOp, ///< The value's user supports a scalar splat operand
+ SplatOpFolded, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index af0eaf3dd5ea0..0579d54f9702a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1004,7 +1004,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
TTI::VectorInstrContext VIC) const {
- if (VIC == TTI::VectorInstrContext::SplatOp)
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded)
return TTI::TCC_Free;
return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
CxtI);
@@ -2575,7 +2575,7 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
// Scalar splat operand can be folded for vector ops that support splatting
// the scalar operand, so the explicit insertelement is free in this context.
if (Opcode == Instruction::InsertElement &&
- VIC == TTI::VectorInstrContext::SplatOp)
+ VIC == TTI::VectorInstrContext::SplatOpFolded)
return TTI::TCC_Free;
// Legalize the type.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 40851d727ee6c..119a94febafec 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22667,7 +22667,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
return TTI->canSplatOperand(UserTE->getOpcode(),
TE->UserTreeIndex.EdgeIdx);
}))
- ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOp);
+ ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOpFolded);
}
}
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
>From eb61ec3b9988f592e57fa7124d69f833ff0c494a Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:20:55 -0700
Subject: [PATCH 05/19] Don't overload getShuffleCost
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 21 +++++-----------
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 ++-
.../AArch64/AArch64TargetTransformInfo.cpp | 11 ++++----
.../AArch64/AArch64TargetTransformInfo.h | 4 ++-
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 ++++-----
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 4 ++-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 12 ++++-----
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 4 ++-
.../Hexagon/HexagonTargetTransformInfo.cpp | 11 ++++----
.../Hexagon/HexagonTargetTransformInfo.h | 4 ++-
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 12 ++++-----
.../Target/PowerPC/PPCTargetTransformInfo.h | 4 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 25 ++++++-------------
.../Target/RISCV/RISCVTargetTransformInfo.h | 11 +++-----
.../SystemZ/SystemZTargetTransformInfo.cpp | 11 ++++----
.../SystemZ/SystemZTargetTransformInfo.h | 4 ++-
.../WebAssemblyTargetTransformInfo.cpp | 4 +--
.../WebAssemblyTargetTransformInfo.h | 4 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 12 ++++-----
llvm/lib/Target/X86/X86TargetTransformInfo.h | 4 ++-
20 files changed, 81 insertions(+), 97 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 2d41a8db947d1..da8e9e0176915 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -776,24 +776,15 @@ class LLVM_ABI TargetTransformInfoImplBase {
return InstructionCost::getInvalid();
}
- virtual InstructionCost
- getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const {
+ virtual InstructionCost getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args = {},
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
- virtual InstructionCost
- getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI, TTI::VectorInstrContext VIC) const {
- (void)VIC;
- return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp,
- Args, CxtI);
- }
-
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cc93f90ff5d8f..6fbe29987d969 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1218,7 +1218,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override {
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
switch (improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp)) {
case TTI::SK_Broadcast:
if (auto *FVT = dyn_cast<FixedVectorType>(SrcTy))
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 341ac1008d7e7..70e6755b23095 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6557,12 +6557,11 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
BinOp, CostKind, FMF);
}
-InstructionCost
-AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost AArch64TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index ddca341bad05b..1a20249357324 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -507,7 +507,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a5ee5fcd76031..30f7bb2f0b03c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1334,13 +1334,11 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
}
}
-InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost GCNTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
if (!isa<FixedVectorType>(SrcTy))
return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,
SubTp);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index df7b6d339e6c2..091ae53c15a50 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -250,7 +250,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 7d4f9ae5b023b..66f5b5aef68b2 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1224,13 +1224,11 @@ InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
return NumOps;
}
-InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost ARMTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 19bd6e7e09366..2d339cbf61aba 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -195,7 +195,9 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool preferInLoopReduction(RecurKind Kind, Type *Ty) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 8ff79f7f4f54d..104c762fc2f1d 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -231,12 +231,11 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
-InstructionCost
-HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost HexagonTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
return 1;
}
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 39517f8c64c88..3f80c9d69bbde 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -124,7 +124,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index b3a54fd4c1ef4..800485d5cfed1 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -617,13 +617,11 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
return Cost * CostFactor;
}
-InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost PPCTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
InstructionCost CostFactor =
vectorCostAdjustmentFactor(Instruction::ShuffleVector, SrcTy, nullptr);
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index c8ec1960b17dd..d2e116b68b60b 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -115,7 +115,9 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 0579d54f9702a..572407ea387bf 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -673,12 +673,14 @@ InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
return FirstSlideCost + SecondSlideCost + MaskCost;
}
-InstructionCost
-RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost RISCVTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded)
+ return TTI::TCC_Free;
+
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
@@ -999,17 +1001,6 @@ RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
SubTp);
}
-InstructionCost RISCVTTIImpl::getShuffleCost(
- TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
- TTI::VectorInstrContext VIC) const {
- if (VIC == TTI::VectorInstrContext::SplatOpFolded)
- return TTI::TCC_Free;
- return getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args,
- CxtI);
-}
-
static unsigned isM1OrSmaller(MVT VT) {
RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 24c0311bf1c2c..a890d68bdc82e 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -179,14 +179,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
- InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI,
- TTI::VectorInstrContext VIC) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 5353c36fd9b97..1464924a007b5 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -772,12 +772,11 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
Args, CxtI);
}
-InstructionCost
-SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost SystemZTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
if (ST->hasVector()) {
unsigned NumVectors = getNumVectorRegs(SrcTy);
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 06f39e585cd47..2f1ac2277250a 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -118,7 +118,9 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const;
unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const;
unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 7d1136bb9beba..5fb26efaed296 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -302,8 +302,8 @@ InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
InstructionCost WebAssemblyTTIImpl::getShuffleCost(
TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
// Canonicalize the ShuffleKind in case optimizations didn't.
// Otherwise, we might end up with the wrong ShuffleKind to match against.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 221f32609b2de..f63024a03fc4f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -90,7 +90,9 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
using BaseT::getVectorInstrCost;
InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 421a2829cb509..8cc1281ed82c2 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1580,13 +1580,11 @@ X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
return InstructionCost::getInvalid();
}
-InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- ArrayRef<int> Mask,
- TTI::TargetCostKind CostKind,
- int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost X86TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 22171f5469d98..c262db0ec8df4 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -81,7 +81,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
>From cf1c39c9f021b42572f8f5fb9319fdd6b36f0e99 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 14:38:14 -0700
Subject: [PATCH 06/19] Use isZeroEltSplatMask
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 119a94febafec..df8b47cbd3ffd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22647,7 +22647,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// than generating the splat vector, the vector instruction may be able to
// take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
// hint to the TTI when costing the insert/shuffle sequence in such cases.
- if (all_of(ReuseMask, [](const int Idx) { return Idx == 0; })) {
+ if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
Value *SplatVal = E->Scalars.front();
if (!isa<VectorType>(SplatVal->getType()) &&
!isa<ExtractElementInst>(SplatVal)) {
>From 12124ad792245cdf2190ac4a59e622bdeaf6e208 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 30 Jul 2026 19:06:51 -0700
Subject: [PATCH 07/19] Add checkForFoldingSplat to ShuffleCostEstimator
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 61 ++++++++++---------
1 file changed, 32 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index df8b47cbd3ffd..d6688b71e55be 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15812,7 +15812,35 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
- void addContextHint(TTI::VectorInstrContext Ctx) { ContextHint = Ctx; }
+ void checkForFoldingSplat(const TreeEntry *E, ArrayRef<int> ReuseMask) {
+ // Backends may have a fast path for splatting scalar operands (i.e. rather
+ // than generating the splat vector, the vector instruction may be able to
+ // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
+ // hint to the TTI when costing the insert/shuffle sequence in such cases.
+ if (!ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size()))
+ return;
+ Value *SplatVal = E->Scalars.front();
+ if (isa<VectorType>(SplatVal->getType()) ||
+ !isa<ExtractElementInst>(SplatVal))
+ return;
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : R.VectorizableTree) {
+ if (R.DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() && "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI.canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ ContextHint = TTI::VectorInstrContext::SplatOpFolded;
+ }
+
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
@@ -21859,7 +21887,6 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
CommonMask.clear();
InVectors.clear();
}
- void addContextHint(TTI::VectorInstrContext Ctx) { (void)Ctx; }
/// Adds 2 input vectors (in form of tree entries) and the mask for their
/// shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
@@ -22643,33 +22670,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- // Backends may have a fast path for splatting scalar operands (i.e. rather
- // than generating the splat vector, the vector instruction may be able to
- // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
- // hint to the TTI when costing the insert/shuffle sequence in such cases.
- if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
- Value *SplatVal = E->Scalars.front();
- if (!isa<VectorType>(SplatVal->getType()) &&
- !isa<ExtractElementInst>(SplatVal)) {
- SmallVector<TreeEntry *> MatchingTEs;
- for (const auto &TE : VectorizableTree) {
- if (DeletedNodes.contains(TE.get()))
- continue;
- if (TE->isGather() && E->isSame(TE->Scalars))
- MatchingTEs.emplace_back(TE.get());
- }
- assert(MatchingTEs.size() &&
- "Ought to at least match with current entry");
- if (all_of(MatchingTEs, [this](auto *TE) {
- auto *UserTE = TE->UserTreeIndex.UserTE;
- if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
- return false;
- return TTI->canSplatOperand(UserTE->getOpcode(),
- TE->UserTreeIndex.EdgeIdx);
- }))
- ShuffleBuilder.addContextHint(TTI::VectorInstrContext::SplatOpFolded);
- }
- }
+ if constexpr (std::is_same_v<BVTy, ShuffleCostEstimator>)
+ ShuffleBuilder.checkForFoldingSplat(E, ReuseMask);
+
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
ShuffleBuilder.add(BV, ReuseMask);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
>From 27c02b92b24bc120370c7a68e9ea70b7d1155b7c Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 01:14:04 -0700
Subject: [PATCH 08/19] Missed adding this before pushing
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d6688b71e55be..b04ea93df8656 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15821,7 +15821,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return;
Value *SplatVal = E->Scalars.front();
if (isa<VectorType>(SplatVal->getType()) ||
- !isa<ExtractElementInst>(SplatVal))
+ isa<ExtractElementInst>(SplatVal))
return;
SmallVector<TreeEntry *> MatchingTEs;
for (const auto &TE : R.VectorizableTree) {
>From 05015a83ac13fc55422c46fac84c785214ec9892 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 01:14:44 -0700
Subject: [PATCH 09/19] Format
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index f31967e265469..dd241ad5f0190 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -190,8 +190,8 @@ enum class VectorInstrContext : uint8_t {
None, ///< The instruction is not folded.
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
- BinaryOp, ///< One of the operands is a binary op.
- SplatOpFolded, ///< The value's user supports a scalar splat operand
+ BinaryOp, ///< One of the operands is a binary op.
+ SplatOpFolded, ///< The value's user supports a scalar splat operand
};
class IntrinsicCostAttributes {
>From dfdf17d21c8fab7a0b6777a3540c84b11a17ae33 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 01:20:13 -0700
Subject: [PATCH 10/19] Fix comment
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index dd241ad5f0190..16ec3eff43401 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,7 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
- SplatOpFolded, ///< The value's user supports a scalar splat operand
+ SplatOpFolded, ///< All of the value's users support splatting the value.
};
class IntrinsicCostAttributes {
>From d7d8e5f04baa2521a23267f42534b375e50c7d59 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 09:54:25 -0700
Subject: [PATCH 11/19] Fix whitespace
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b04ea93df8656..651799b53abf1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15993,7 +15993,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
getAllOnesValue(*R.DL, ScalarTy->getScalarType()));
}
InstructionCost createFreeze(InstructionCost Cost) { return Cost; }
-
/// Finalize emission of the shuffles.
InstructionCost finalize(
ArrayRef<int> ExtMask,
>From e636df5915593d2619e7ecf23eee6bb511ef35f8 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 10:11:51 -0700
Subject: [PATCH 12/19] Add assertions
---
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 572407ea387bf..c57da68a298ab 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -678,6 +678,10 @@ InstructionCost RISCVTTIImpl::getShuffleCost(
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
TTI::VectorInstrContext VIC) const {
+ assert((improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp) ==
+ TTI::SK_Broadcast ||
+ VIC != TTI::VectorInstrContext::SplatOpFolded) &&
+ "Must be SK_Broadcast if a splat operation");
if (VIC == TTI::VectorInstrContext::SplatOpFolded)
return TTI::TCC_Free;
@@ -2566,8 +2570,10 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
// Scalar splat operand can be folded for vector ops that support splatting
// the scalar operand, so the explicit insertelement is free in this context.
if (Opcode == Instruction::InsertElement &&
- VIC == TTI::VectorInstrContext::SplatOpFolded)
+ VIC == TTI::VectorInstrContext::SplatOpFolded) {
+ assert(Index == 0 && "SplatOpFolded sequence must insert into lane 0");
return TTI::TCC_Free;
+ }
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
>From 9399336195ce2f1fe69c90ef87f38565565d0c91 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 09:21:36 -0700
Subject: [PATCH 13/19] Consider ST->sinkSplatOperands() where costing splatted
operands
---
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index c57da68a298ab..5c8a911ed915b 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -682,7 +682,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(
TTI::SK_Broadcast ||
VIC != TTI::VectorInstrContext::SplatOpFolded) &&
"Must be SK_Broadcast if a splat operation");
- if (VIC == TTI::VectorInstrContext::SplatOpFolded)
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded && ST->sinkSplatOperands())
return TTI::TCC_Free;
assert((Mask.empty() || DstTy->isScalableTy() ||
@@ -2570,7 +2570,8 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
// Scalar splat operand can be folded for vector ops that support splatting
// the scalar operand, so the explicit insertelement is free in this context.
if (Opcode == Instruction::InsertElement &&
- VIC == TTI::VectorInstrContext::SplatOpFolded) {
+ VIC == TTI::VectorInstrContext::SplatOpFolded &&
+ ST->sinkSplatOperands()) {
assert(Index == 0 && "SplatOpFolded sequence must insert into lane 0");
return TTI::TCC_Free;
}
>From 127df10605d85dca8e64f8085c68add9e08418b0 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 11:46:54 -0700
Subject: [PATCH 14/19] [SLP] Store ContextHint's with InVectors
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 218 +++++++++++-------
1 file changed, 132 insertions(+), 86 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 651799b53abf1..39ac30edf8f88 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15118,7 +15118,19 @@ void BoUpSLP::transformNodes() {
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
bool IsFinalized = false;
SmallVector<int> CommonMask;
- SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
+
+ // Each input vector can be tagged with extra context information
+ struct InVector {
+ PointerUnion<Value *, const TreeEntry *> Vec = nullptr;
+ TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None;
+
+ InVector(Value *Vec)
+ : Vec(Vec), Ctx(TargetTransformInfo::VectorInstrContext::None) {}
+ InVector(const TreeEntry *Vec)
+ : Vec(Vec), Ctx(TargetTransformInfo::VectorInstrContext::None) {}
+ };
+ SmallVector<InVector, 2> InVectors;
/// Captures the original scalar VL of a single, "clean" gather() call so
/// the values can be forwarded as the Args operand to getShuffleCost() for
/// the final permutation in finalize(). This lets the target cost model
@@ -15143,7 +15155,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
/// May help better estimate the cost if same nodes must be permuted + allows
/// to move most of the long shuffles cost estimation to TTI.
bool SameNodesEstimated = true;
- TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None;
static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
if (Ty->getScalarType()->isPointerTy()) {
@@ -15159,7 +15170,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return Constant::getAllOnesValue(Ty);
}
- InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) {
+ InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root,
+ TTI::VectorInstrContext ContextHint) {
if ((!Root && allConstant(VL)) || all_of(VL, IsaPred<UndefValue>))
return TTI::TCC_Free;
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
@@ -15200,6 +15212,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
/*Index=*/0, /*SubTp=*/nullptr,
/*Args=*/*It, ContextHint);
}
+ assert(ContextHint != TTI::VectorInstrContext::SplatOpFolded &&
+ "Expected SplatOpFolder to be a splat");
return GatherCost +
(all_of(Gathers, IsaPred<UndefValue>)
? TTI::TCC_Free
@@ -15348,9 +15362,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// sub-Mask into the CommonMask to estimate it later and avoid double cost
// estimation.
if ((InVectors.size() == 2 &&
- cast<const TreeEntry *>(InVectors.front()) == &E1 &&
- cast<const TreeEntry *>(InVectors.back()) == E2) ||
- (!E2 && cast<const TreeEntry *>(InVectors.front()) == &E1)) {
+ cast<const TreeEntry *>(InVectors.front().Vec) == &E1 &&
+ cast<const TreeEntry *>(InVectors.back().Vec) == E2) ||
+ (!E2 && cast<const TreeEntry *>(InVectors.front().Vec) == &E1)) {
unsigned Limit = getNumElems(Mask.size(), SliceSize, Part);
assert(all_of(ArrayRef(CommonMask).slice(Part * SliceSize, Limit),
[](int Idx) { return Idx == PoisonMaskElem; }) &&
@@ -15361,50 +15375,57 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
// Found non-matching nodes - need to estimate the cost for the matched
// and transform mask.
- Cost += createShuffle(InVectors.front(),
- InVectors.size() == 1 ? nullptr : InVectors.back(),
- CommonMask);
+ if (InVectors.size() == 1)
+ Cost += createShuffle(InVectors.front().Vec, nullptr, CommonMask, {},
+ InVectors.front().Ctx);
+ else
+ Cost += createShuffle(InVectors.front().Vec, InVectors.back().Vec,
+ CommonMask, {}, InVectors.front().Ctx,
+ InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else if (InVectors.size() == 2) {
- Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask);
+ Cost +=
+ createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
+ {}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
SameNodesEstimated = false;
if (!E2 && InVectors.size() == 1) {
unsigned VF = E1.getVectorFactor();
- if (Value *V1 = dyn_cast<Value *>(InVectors.front())) {
+ if (Value *V1 = InVectors.front().Vec.dyn_cast<Value *>()) {
VF = std::max(VF, getVF(V1));
} else {
- const auto *E = cast<const TreeEntry *>(InVectors.front());
+ const auto *E = cast<const TreeEntry *>(InVectors.front().Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
- Cost += createShuffle(InVectors.front(), &E1, CommonMask);
+ Cost += createShuffle(InVectors.front().Vec, &E1, CommonMask, {},
+ InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else {
auto P = InVectors.front();
- Cost += createShuffle(&E1, E2, Mask);
+ Cost += createShuffle(&E1, E2, Mask, {}, TTI::VectorInstrContext::None);
unsigned VF = Mask.size();
- if (Value *V1 = dyn_cast<Value *>(P)) {
+ if (Value *V1 = dyn_cast<Value *>(P.Vec)) {
VF = std::max(VF,
getNumElements(V1->getType()));
} else {
- const auto *E = cast<const TreeEntry *>(P);
+ const auto *E = cast<const TreeEntry *>(P.Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx + (InVectors.empty() ? 0 : VF);
- Cost += createShuffle(P, InVectors.front(), CommonMask);
+ Cost += createShuffle(P.Vec, InVectors.front().Vec, CommonMask, {}, P.Ctx,
+ InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
}
class ShuffleCostBuilder {
const TargetTransformInfo &TTI;
- const TTI::VectorInstrContext ContextHint;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
int Index = -1;
@@ -15416,9 +15437,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
public:
- ShuffleCostBuilder(const TargetTransformInfo &TTI,
- TTI::VectorInstrContext ContextHint)
- : TTI(TTI), ContextHint(ContextHint) {}
+ ShuffleCostBuilder(const TargetTransformInfo &TTI) : TTI(TTI) {}
~ShuffleCostBuilder() = default;
InstructionCost createShuffleVector(Value *V1, Value *,
ArrayRef<int> Mask) const {
@@ -15430,8 +15449,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(V1->getType()), Mask);
}
- InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask,
- ArrayRef<Value *> VL) const {
+ InstructionCost
+ createShuffleVector(Value *V1, ArrayRef<int> Mask, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext ContextHint =
+ TTI::VectorInstrContext::None) const {
// Empty mask or identity mask are free.
unsigned VF =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
@@ -15455,8 +15476,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
InstructionCost
createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
const PointerUnion<Value *, const TreeEntry *> &P2,
- ArrayRef<int> Mask, ArrayRef<Value *> VL = {}) {
- ShuffleCostBuilder Builder(TTI, ContextHint);
+ ArrayRef<int> Mask, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext Ctx1 = TTI::VectorInstrContext::None,
+ TTI::VectorInstrContext Ctx2 = TTI::VectorInstrContext::None) {
+ ShuffleCostBuilder Builder(TTI);
SmallVector<int> CommonMask(Mask);
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
unsigned CommonVF = Mask.size();
@@ -15662,8 +15685,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
if (InVectors.size() == 2)
InVectors.pop_back();
+ assert(Ctx2 == TTI::VectorInstrContext::None &&
+ "Did not expect context for second vector");
+ assert((Ctx1 == TTI::VectorInstrContext::None || P2.isNull()) &&
+ "Only support context for single vector shuffle");
return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
- V1, V2, CommonMask, Builder, ScalarTy, VL);
+ V1, V2, CommonMask, Builder, ScalarTy, VL, Ctx1);
}
public:
@@ -15812,34 +15839,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
- void checkForFoldingSplat(const TreeEntry *E, ArrayRef<int> ReuseMask) {
- // Backends may have a fast path for splatting scalar operands (i.e. rather
- // than generating the splat vector, the vector instruction may be able to
- // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
- // hint to the TTI when costing the insert/shuffle sequence in such cases.
- if (!ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size()))
- return;
- Value *SplatVal = E->Scalars.front();
- if (isa<VectorType>(SplatVal->getType()) ||
- isa<ExtractElementInst>(SplatVal))
- return;
- SmallVector<TreeEntry *> MatchingTEs;
- for (const auto &TE : R.VectorizableTree) {
- if (R.DeletedNodes.contains(TE.get()))
- continue;
- if (TE->isGather() && E->isSame(TE->Scalars))
- MatchingTEs.emplace_back(TE.get());
- }
- assert(MatchingTEs.size() && "Ought to at least match with current entry");
- if (all_of(MatchingTEs, [this](auto *TE) {
- auto *UserTE = TE->UserTreeIndex.UserTE;
- if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
- return false;
- return TTI.canSplatOperand(UserTE->getOpcode(),
- TE->UserTreeIndex.EdgeIdx);
- }))
- ContextHint = TTI::VectorInstrContext::SplatOpFolded;
- }
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
@@ -15894,7 +15893,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
if (P.value() == PoisonMaskElem)
return Mask[P.index()] == PoisonMaskElem;
auto *EI = cast<ExtractElementInst>(
- cast<const TreeEntry *>(InVectors.front())
+ cast<const TreeEntry *>(InVectors.front().Vec)
->getOrdered(P.index()));
return EI->getVectorOperand() == V1 ||
EI->getVectorOperand() == V2;
@@ -15902,7 +15901,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
"Expected extractelement vectors.");
}
/// Adds another one input vector and the mask for the shuffling.
- void add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false) {
+ void
+ add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false,
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
+ assert(
+ (InVectors.empty() || ContextHint == TTI::VectorInstrContext::None) &&
+ "Only expected hint with single input shuffle");
if (BVValues && !isa<Constant>(V1))
BVValues.reset();
if (InVectors.empty()) {
@@ -15910,15 +15914,16 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
"Expected empty input mask/vectors.");
CommonMask.assign(Mask.begin(), Mask.end());
InVectors.assign(1, V1);
+ InVectors.back().Ctx = ContextHint;
return;
}
if (ForExtracts) {
// No need to add vectors here, already handled them in adjustExtracts.
- assert(InVectors.size() == 1 && isa<const TreeEntry *>(InVectors[0]) &&
- !CommonMask.empty() &&
+ assert(InVectors.size() == 1 &&
+ isa<const TreeEntry *>(InVectors[0].Vec) && !CommonMask.empty() &&
all_of(enumerate(CommonMask),
[&](auto P) {
- Value *Scalar = cast<const TreeEntry *>(InVectors[0])
+ Value *Scalar = cast<const TreeEntry *>(InVectors[0].Vec)
->getOrdered(P.index());
if (P.value() == PoisonMaskElem)
return P.value() == Mask[P.index()] ||
@@ -15935,25 +15940,28 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
"Expected only tree entries from extracts/reused buildvectors.");
unsigned VF = getVF(V1);
if (InVectors.size() == 2) {
- Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask);
+ Cost +=
+ createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
+ {}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
VF = std::max<unsigned>(VF, CommonMask.size());
} else if (const auto *InTE =
- InVectors.front().dyn_cast<const TreeEntry *>()) {
+ InVectors.front().Vec.dyn_cast<const TreeEntry *>()) {
VF = std::max(VF, InTE->getVectorFactor());
} else {
- VF = std::max(
- VF, cast<FixedVectorType>(cast<Value *>(InVectors.front())->getType())
- ->getNumElements());
+ VF = std::max(VF, cast<FixedVectorType>(
+ cast<Value *>(InVectors.front().Vec)->getType())
+ ->getNumElements());
}
InVectors.push_back(V1);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
}
- Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
- Value *Root = nullptr) {
- Cost += getBuildVectorCost(VL, Root);
+ Value *
+ gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, Value *Root = nullptr,
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
+ Cost += getBuildVectorCost(VL, Root, ContextHint);
if (BVValues) {
if (BVValues->empty() && InVectors.empty())
BVValues->assign(VL.begin(), VL.end());
@@ -16003,11 +16011,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Action = {}) {
IsFinalized = true;
if (Action) {
- const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
+ const PointerUnion<Value *, const TreeEntry *> &Vec =
+ InVectors.front().Vec;
if (InVectors.size() == 2)
- Cost += createShuffle(Vec, InVectors.back(), CommonMask);
+ Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
+ InVectors.front().Ctx, InVectors.back().Ctx);
else
- Cost += createShuffle(Vec, nullptr, CommonMask);
+ Cost +=
+ createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
assert(VF > 0 &&
"Expected vector length for the final value before action.");
@@ -16019,11 +16030,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
InVectors.front() = V;
}
if (!SubVectors.empty()) {
- const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
+ const PointerUnion<Value *, const TreeEntry *> &Vec =
+ InVectors.front().Vec;
if (InVectors.size() == 2)
- Cost += createShuffle(Vec, InVectors.back(), CommonMask);
+ Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
+ InVectors.front().Ctx, InVectors.back().Ctx);
else
- Cost += createShuffle(Vec, nullptr, CommonMask);
+ Cost +=
+ createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
// Add subvectors permutation cost.
if (!SubVectorsMask.empty()) {
@@ -16094,10 +16108,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
ArrayRef<Value *> VL;
if (BVValues)
VL = *BVValues;
- return Cost +
- createShuffle(InVectors.front(),
- InVectors.size() == 2 ? InVectors.back() : nullptr,
- CommonMask, VL);
+ if (InVectors.size() == 2)
+ return Cost + createShuffle(InVectors.front().Vec, InVectors.back().Vec,
+ CommonMask, VL, InVectors.front().Ctx,
+ InVectors.back().Ctx);
+ return Cost + createShuffle(InVectors.front().Vec, nullptr, CommonMask, VL,
+ InVectors.front().Ctx);
}
~ShuffleCostEstimator() {
@@ -21934,7 +21950,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
InVectors.push_back(V1);
}
/// Adds another one input vector and the mask for the shuffling.
- void add(Value *V1, ArrayRef<int> Mask, bool = false) {
+ void add(Value *V1, ArrayRef<int> Mask, bool = false,
+ TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
assert(isa<FixedVectorType>(V1->getType()) &&
"castToScalarTyElem expects V1 to be FixedVectorType");
V1 = castToScalarTyElem(V1);
@@ -21993,7 +22010,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
add(V1, NewMask);
}
Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
- Value *Root = nullptr) {
+ Value *Root = nullptr,
+ TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
return R.gather(VL, Root, ScalarTy,
[&](Value *V1, Value *V2, ArrayRef<int> Mask) {
return createShuffle(V1, V2, Mask);
@@ -22009,7 +22027,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
ArrayRef<int> SubVectorsMask, unsigned VF = 0,
function_ref<void(Value *&, SmallVectorImpl<int> &,
function_ref<Value *(Value *, Value *, ArrayRef<int>)>)>
- Action = {}) {
+ Action = {},
+ TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
IsFinalized = true;
if (Action) {
Value *Vec = InVectors.front();
@@ -22669,13 +22688,40 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- if constexpr (std::is_same_v<BVTy, ShuffleCostEstimator>)
- ShuffleBuilder.checkForFoldingSplat(E, ReuseMask);
-
- Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
- ShuffleBuilder.add(BV, ReuseMask);
+ // Backends may have a fast path for splatting scalar operands (i.e. rather
+ // than generating the splat vector, the vector instruction may be able to
+ // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
+ // hint to the TTI when costing the insert/shuffle sequence in such cases.
+ auto ContextHint = TTI::VectorInstrContext::None;
+
+ if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
+ Value *SplatVal = E->Scalars.front();
+ if (!isa<VectorType>(SplatVal->getType()) &&
+ !isa<ExtractElementInst>(SplatVal)) {
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() &&
+ "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI->canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ ContextHint = TTI::VectorInstrContext::SplatOpFolded;
+ }
+ }
+ Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(),
+ /*Root*/ nullptr, ContextHint);
+ ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
- SubVectorsMask);
+ SubVectorsMask, /*VF*/ 0, /*Action*/ {});
} else {
// Gather all constants.
SmallVector<int> Mask(GatheredScalars.size(), PoisonMaskElem);
>From 3b5a7c612fbe3187437739624b0caaa2cc8b834b Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 20:07:41 -0700
Subject: [PATCH 15/19] Merge context hints when appropriate
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 +++-----
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp | 6 ++++++
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h | 8 +++++++-
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 39ac30edf8f88..7e9ce3c4a6448 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15685,12 +15685,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
if (InVectors.size() == 2)
InVectors.pop_back();
- assert(Ctx2 == TTI::VectorInstrContext::None &&
- "Did not expect context for second vector");
- assert((Ctx1 == TTI::VectorInstrContext::None || P2.isNull()) &&
- "Only support context for single vector shuffle");
+ TTI::VectorInstrContext Ctx =
+ P2.isNull() ? Ctx1 : combineVectorInstrContexts(Ctx1, Ctx2);
return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
- V1, V2, CommonMask, Builder, ScalarTy, VL, Ctx1);
+ V1, V2, CommonMask, Builder, ScalarTy, VL, Ctx);
}
public:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 586753e563555..2ac47536a0db2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -676,4 +676,10 @@ Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
}
}
+TargetTransformInfo::VectorInstrContext
+combineVectorInstrContexts(TargetTransformInfo::VectorInstrContext Ctx1,
+ TargetTransformInfo::VectorInstrContext Ctx2) {
+ return Ctx1 == Ctx2 ? Ctx1 : TargetTransformInfo::VectorInstrContext::None;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index c8a3204649275..60a9367378e9a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -20,6 +20,7 @@
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Intrinsics.h"
#include <optional>
@@ -30,7 +31,6 @@ class Constant;
class DataLayout;
class Instruction;
class TargetLibraryInfo;
-class TargetTransformInfo;
class Type;
class Value;
} // namespace llvm
@@ -302,6 +302,12 @@ SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF);
/// unlike the plain opcode.
Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode);
+/// Combines 2 context hints into a single value. If both are equal, keep the
+/// shared context, otherwise fall back to no specific context.
+TargetTransformInfo::VectorInstrContext
+combineVectorInstrContexts(TargetTransformInfo::VectorInstrContext Ctx1,
+ TargetTransformInfo::VectorInstrContext Ctx2);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
>From 82a13bad54a7d2d070361ba2bdff9db3da3b823f Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 20:14:10 -0700
Subject: [PATCH 16/19] Extract getBuildVectorContextHint()
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 66 +++++++++++--------
1 file changed, 37 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e9ce3c4a6448..3da697b44df50 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3233,6 +3233,13 @@ class slpvectorizer::BoUpSLP {
/// \p E.
Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
+ /// Pass a hint to the TTI when building vectors for special use cases
+ /// Backends may have a fast path for splatting scalar operands (i.e. rather
+ /// than generating the splat vector, the vector instruction may be able to
+ /// take a scalar operand), for example RISCV vfoo.vx instructions.
+ TargetTransformInfo::VectorInstrContext
+ getBuildVectorContextHint(const TreeEntry *E, ArrayRef<int> ReuseMask);
+
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
@@ -22135,6 +22142,35 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
return vectorizeTree(getOperandEntry(E, NodeIdx));
}
+TargetTransformInfo::VectorInstrContext
+BoUpSLP::getBuildVectorContextHint(const TreeEntry *E,
+ ArrayRef<int> ReuseMask) {
+ if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
+ Value *SplatVal = E->Scalars.front();
+ if (!isa<VectorType>(SplatVal->getType()) &&
+ !isa<ExtractElementInst>(SplatVal)) {
+ SmallVector<TreeEntry *> MatchingTEs;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (TE->isGather() && E->isSame(TE->Scalars))
+ MatchingTEs.emplace_back(TE.get());
+ }
+ assert(MatchingTEs.size() &&
+ "Ought to at least match with current entry");
+ if (all_of(MatchingTEs, [this](auto *TE) {
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ return TTI->canSplatOperand(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }))
+ return TTI::VectorInstrContext::SplatOpFolded;
+ }
+ }
+ return TTI::VectorInstrContext::None;
+}
+
template <typename BVTy, typename ResTy, typename... Args>
ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
Args &...Params) {
@@ -22686,35 +22722,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- // Backends may have a fast path for splatting scalar operands (i.e. rather
- // than generating the splat vector, the vector instruction may be able to
- // take a scalar operand), for example RISCV vfoo.vx instructions. Pass a
- // hint to the TTI when costing the insert/shuffle sequence in such cases.
- auto ContextHint = TTI::VectorInstrContext::None;
-
- if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
- Value *SplatVal = E->Scalars.front();
- if (!isa<VectorType>(SplatVal->getType()) &&
- !isa<ExtractElementInst>(SplatVal)) {
- SmallVector<TreeEntry *> MatchingTEs;
- for (const auto &TE : VectorizableTree) {
- if (DeletedNodes.contains(TE.get()))
- continue;
- if (TE->isGather() && E->isSame(TE->Scalars))
- MatchingTEs.emplace_back(TE.get());
- }
- assert(MatchingTEs.size() &&
- "Ought to at least match with current entry");
- if (all_of(MatchingTEs, [this](auto *TE) {
- auto *UserTE = TE->UserTreeIndex.UserTE;
- if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
- return false;
- return TTI->canSplatOperand(UserTE->getOpcode(),
- TE->UserTreeIndex.EdgeIdx);
- }))
- ContextHint = TTI::VectorInstrContext::SplatOpFolded;
- }
- }
+ auto ContextHint = getBuildVectorContextHint(E, ReuseMask);
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(),
/*Root*/ nullptr, ContextHint);
ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint);
>From 2b54ca15fffa255a382446a47d1cf9b950892b09 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 20:15:13 -0700
Subject: [PATCH 17/19] Don't drop ContextHint
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3da697b44df50..05734f1995bde 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15132,10 +15132,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
TargetTransformInfo::VectorInstrContext Ctx =
TargetTransformInfo::VectorInstrContext::None;
- InVector(Value *Vec)
- : Vec(Vec), Ctx(TargetTransformInfo::VectorInstrContext::None) {}
- InVector(const TreeEntry *Vec)
- : Vec(Vec), Ctx(TargetTransformInfo::VectorInstrContext::None) {}
+ InVector(Value *Vec, TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None)
+ : Vec(Vec), Ctx(Ctx) {}
+ InVector(const TreeEntry *Vec,
+ TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None)
+ : Vec(Vec), Ctx(Ctx) {}
};
SmallVector<InVector, 2> InVectors;
/// Captures the original scalar VL of a single, "clean" gather() call so
@@ -15909,17 +15912,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
void
add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false,
TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
- assert(
- (InVectors.empty() || ContextHint == TTI::VectorInstrContext::None) &&
- "Only expected hint with single input shuffle");
if (BVValues && !isa<Constant>(V1))
BVValues.reset();
if (InVectors.empty()) {
assert(CommonMask.empty() && !ForExtracts &&
"Expected empty input mask/vectors.");
CommonMask.assign(Mask.begin(), Mask.end());
- InVectors.assign(1, V1);
- InVectors.back().Ctx = ContextHint;
+ InVectors.assign(1, InVector(V1, ContextHint));
return;
}
if (ForExtracts) {
@@ -15958,7 +15957,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<Value *>(InVectors.front().Vec)->getType())
->getNumElements());
}
- InVectors.push_back(V1);
+ InVectors.emplace_back(V1, ContextHint);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
>From 8c4ac9caf5c96d467006c45647f5e2b33688108c Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 20:15:24 -0700
Subject: [PATCH 18/19] Remove newline
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 05734f1995bde..ff8e07ab29475 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15847,7 +15847,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
VectorizedVals.clear();
SameNodesEstimated = true;
}
-
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
>From f1d38618108937d17ad58d246fd6bf18fda0aff0 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Tue, 11 Aug 2026 12:54:56 -0700
Subject: [PATCH 19/19] Refactor to create TTI::getBuildVectorContextHint()
---
.../llvm/Analysis/TargetTransformInfo.h | 17 ++++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 25 +++++++++
.../Transforms/Vectorize/SLPVectorizer.cpp | 56 +++++++------------
3 files changed, 61 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 16ec3eff43401..9d2d86a4a30ec 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -24,6 +24,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/Uniformity.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InterestingMemoryOperand.h"
@@ -1075,10 +1076,26 @@ class TargetTransformInfo {
using VectorInstrContext = llvm::VectorInstrContext;
+ /// Stores information about the uses of a build vector
+ struct BuildVectorUseOp {
+ unsigned Opcode;
+ int OperandIndex;
+ };
+
/// Calculates a VectorInstrContext from \p I.
LLVM_ABI static VectorInstrContext
getVectorInstrContextHint(const Instruction *I);
+ /// Calculates a VectorInstrContext for buildvector-like gather sequences.
+ ///
+ /// \p GatherUserOps must collect all users of \p Scalars relevant for
+ /// determining whether a splat can be folded as a scalar operand. It returns
+ /// false if those users cannot be gathered in the required form.
+ LLVM_ABI VectorInstrContext getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps)
+ const;
+
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
/// are set if the demanded result elements need to be inserted and/or
/// extracted from vectors. The involved values may be passed in VL if
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 95b338946bc70..f3c37491c44be 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -666,6 +666,31 @@ TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) {
return VectorInstrContext::None;
}
+TargetTransformInfo::VectorInstrContext
+TargetTransformInfo::getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps)
+ const {
+ if (Scalars.empty() ||
+ !ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))
+ return VectorInstrContext::None;
+
+ Value *SplatVal = Scalars.front();
+ if (isa<VectorType>(SplatVal->getType()) || isa<ExtractElementInst>(SplatVal))
+ return VectorInstrContext::None;
+
+ SmallVector<BuildVectorUseOp, 4> UserOps;
+ if (!GatherUseOps(UserOps) || UserOps.empty())
+ return VectorInstrContext::None;
+
+ if (all_of(UserOps, [this](const BuildVectorUseOp &UserOp) {
+ return canSplatOperand(UserOp.Opcode, UserOp.OperandIndex);
+ }))
+ return VectorInstrContext::SplatOpFolded;
+
+ return VectorInstrContext::None;
+}
+
InstructionCost TargetTransformInfo::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index ff8e07ab29475..b6cecf46517f5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3233,13 +3233,6 @@ class slpvectorizer::BoUpSLP {
/// \p E.
Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
- /// Pass a hint to the TTI when building vectors for special use cases
- /// Backends may have a fast path for splatting scalar operands (i.e. rather
- /// than generating the splat vector, the vector instruction may be able to
- /// take a scalar operand), for example RISCV vfoo.vx instructions.
- TargetTransformInfo::VectorInstrContext
- getBuildVectorContextHint(const TreeEntry *E, ArrayRef<int> ReuseMask);
-
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
@@ -22140,35 +22133,6 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
return vectorizeTree(getOperandEntry(E, NodeIdx));
}
-TargetTransformInfo::VectorInstrContext
-BoUpSLP::getBuildVectorContextHint(const TreeEntry *E,
- ArrayRef<int> ReuseMask) {
- if (ShuffleVectorInst::isZeroEltSplatMask(ReuseMask, ReuseMask.size())) {
- Value *SplatVal = E->Scalars.front();
- if (!isa<VectorType>(SplatVal->getType()) &&
- !isa<ExtractElementInst>(SplatVal)) {
- SmallVector<TreeEntry *> MatchingTEs;
- for (const auto &TE : VectorizableTree) {
- if (DeletedNodes.contains(TE.get()))
- continue;
- if (TE->isGather() && E->isSame(TE->Scalars))
- MatchingTEs.emplace_back(TE.get());
- }
- assert(MatchingTEs.size() &&
- "Ought to at least match with current entry");
- if (all_of(MatchingTEs, [this](auto *TE) {
- auto *UserTE = TE->UserTreeIndex.UserTE;
- if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
- return false;
- return TTI->canSplatOperand(UserTE->getOpcode(),
- TE->UserTreeIndex.EdgeIdx);
- }))
- return TTI::VectorInstrContext::SplatOpFolded;
- }
- }
- return TTI::VectorInstrContext::None;
-}
-
template <typename BVTy, typename ResTy, typename... Args>
ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
Args &...Params) {
@@ -22720,7 +22684,25 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- auto ContextHint = getBuildVectorContextHint(E, ReuseMask);
+ auto GatherUserOps = [&](SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) {
+ bool HasMatches = false;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (!(TE->isGather() && E->isSame(TE->Scalars)))
+ continue;
+ HasMatches = true;
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ UserOps.push_back(
+ {UserTE->getOpcode(), static_cast<int>(TE->UserTreeIndex.EdgeIdx)});
+ }
+ assert(HasMatches && "Ought to at least match with current entry");
+ return HasMatches;
+ };
+ TargetTransformInfo::VectorInstrContext ContextHint =
+ TTI->getBuildVectorContextHint(ReuseMask, E->Scalars, GatherUserOps);
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(),
/*Root*/ nullptr, ContextHint);
ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint);
More information about the llvm-commits
mailing list