[llvm] [SLP] Reland: More accurately cost RISCV scalar splats (PR #224766)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 11:52:13 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/224766
>From f7a480de11c77ff705ec7a2ad55d348eb4872f3e Mon Sep 17 00:00:00 2001
From: Ryan Buchner <rbuchner at qti.qualcomm.com>
Date: Wed, 16 Sep 2026 15:24:30 -0700
Subject: [PATCH 1/3] [SLP] More accurately cost RISCV scalar splats (#213104)
Backends may have a fast path for splatting scalar operands (i.e. rather
than generating the splat vector, the vector instruction may be able to
take a scalar operand), for example RISCV `vfoo.vx` instructions. Pass a
hint to the TTI when costing the insert/shuffle sequence in such cases.
Fixes #212413.
Assisted By: Codex
---
.../llvm/Analysis/TargetTransformInfo.h | 31 ++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 19 +-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 21 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 11 +-
.../AArch64/AArch64TargetTransformInfo.h | 4 +-
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 12 +-
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 4 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 12 +-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 4 +-
.../Hexagon/HexagonTargetTransformInfo.cpp | 11 +-
.../Hexagon/HexagonTargetTransformInfo.h | 4 +-
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 12 +-
.../Target/PowerPC/PPCTargetTransformInfo.h | 4 +-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 48 ++++-
.../Target/RISCV/RISCVTargetTransformInfo.h | 10 +-
.../SystemZ/SystemZTargetTransformInfo.cpp | 11 +-
.../SystemZ/SystemZTargetTransformInfo.h | 4 +-
.../WebAssemblyTargetTransformInfo.cpp | 4 +-
.../WebAssemblyTargetTransformInfo.h | 4 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 12 +-
llvm/lib/Target/X86/X86TargetTransformInfo.h | 4 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 188 +++++++++++++-----
.../SLPVectorizer/SLPCostAnalysis.cpp | 15 +-
.../Vectorize/SLPVectorizer/SLPCostAnalysis.h | 7 +-
.../RISCV/check-node-without-vector-user.ll | 42 +---
.../RISCV/folded-broadcast-cost.ll | 4 +-
.../remark-zext-incoming-for-neg-icmp.ll | 2 +-
.../RISCV/reordered-buildvector-scalars.ll | 8 +-
29 files changed, 340 insertions(+), 176 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index f1731bc364555..0b183954198dd 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -24,6 +24,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/Uniformity.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InterestingMemoryOperand.h"
@@ -190,7 +191,8 @@ enum class VectorInstrContext : uint8_t {
None, ///< The instruction is not folded.
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
- BinaryOp, ///< One of the operands is a binary op.
+ BinaryOp, ///< One of the operands is a binary op.
+ SplatOpFolded, ///< All of the value's users support splatting the value.
};
class IntrinsicCostAttributes {
@@ -1072,10 +1074,34 @@ class TargetTransformInfo {
using VectorInstrContext = llvm::VectorInstrContext;
+ /// Combines 2 context hints into a single value. If both are equal, keep the
+ /// shared context, otherwise fall back to no specific context.
+ LLVM_ABI static TargetTransformInfo::VectorInstrContext
+ combineVectorInstrContexts(TargetTransformInfo::VectorInstrContext Ctx1,
+ TargetTransformInfo::VectorInstrContext Ctx2);
+
+ /// Stores information about the uses of a build vector
+ struct BuildVectorUseOp {
+ unsigned Opcode;
+ int OperandIndex;
+ BuildVectorUseOp(unsigned Opcode, int OperandIndex)
+ : Opcode(Opcode), OperandIndex(OperandIndex) {}
+ };
+
/// Calculates a VectorInstrContext from \p I.
LLVM_ABI static VectorInstrContext
getVectorInstrContextHint(const Instruction *I);
+ /// Calculates a VectorInstrContext for buildvector-like gather sequences.
+ ///
+ /// \p GatherUserOps must collect all users of \p Scalars relevant for
+ /// determining whether a splat can be folded as a scalar operand. It returns
+ /// false if those users cannot be gathered in the required form.
+ LLVM_ABI VectorInstrContext getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps)
+ const;
+
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
/// are set if the demanded result elements need to be inserted and/or
/// extracted from vectors. The involved values may be passed in VL if
@@ -1562,7 +1588,8 @@ class TargetTransformInfo {
ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask = {}, int Index = 0,
VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// Represents a hint about the context in which a cast is used.
///
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 84cb3a6e664b9..5103d3dce25b2 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -773,11 +773,12 @@ class LLVM_ABI TargetTransformInfoImplBase {
return InstructionCost::getInvalid();
}
- virtual InstructionCost
- getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
- TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const {
+ virtual InstructionCost getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args = {},
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
@@ -1140,6 +1141,14 @@ class LLVM_ABI TargetTransformInfoImplBase {
return true;
}
+ virtual TargetTransformInfo::VectorInstrContext getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<
+ bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
+ GatherUseOps) const {
+ return TargetTransformInfo::VectorInstrContext::None;
+ }
+
virtual bool isElementTypeLegalForScalableVector(Type *Ty) const {
return true;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 9723016ca23b4..9895bc94478b4 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1226,7 +1226,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override {
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
switch (improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp)) {
case TTI::SK_Broadcast:
if (auto *FVT = dyn_cast<FixedVectorType>(SrcTy))
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index a9f76a735a48c..afd6f895039e0 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -643,6 +643,13 @@ bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField(
return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
}
+TargetTransformInfo::VectorInstrContext
+TargetTransformInfo::combineVectorInstrContexts(
+ TargetTransformInfo::VectorInstrContext Ctx1,
+ TargetTransformInfo::VectorInstrContext Ctx2) {
+ return Ctx1 == Ctx2 ? Ctx1 : TargetTransformInfo::VectorInstrContext::None;
+}
+
TargetTransformInfo::VectorInstrContext
TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) {
if (!I)
@@ -662,6 +669,14 @@ TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) {
return VectorInstrContext::None;
}
+TargetTransformInfo::VectorInstrContext
+TargetTransformInfo::getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<bool(SmallVectorImpl<BuildVectorUseOp> &)> GatherUseOps)
+ const {
+ return TTIImpl->getBuildVectorContextHint(Mask, Scalars, GatherUseOps);
+}
+
InstructionCost TargetTransformInfo::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
@@ -1037,15 +1052,15 @@ InstructionCost TargetTransformInfo::getAltInstrCost(
InstructionCost TargetTransformInfo::getShuffleCost(
ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
"Expected the same scalar types");
InstructionCost Cost = TTIImpl->getShuffleCost(
- Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp, Args, CxtI);
+ Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp, Args, CxtI, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8e0b88dbad0eb..272bd8b0af64d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6948,12 +6948,11 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
BinOp, CostKind, FMF);
}
-InstructionCost
-AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost AArch64TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index dc840f5c7565f..f8f771525150e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -516,7 +516,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a7556278b7e0d..740042db01a51 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1379,13 +1379,11 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
}
}
-InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost GCNTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
if (!isa<FixedVectorType>(SrcTy))
return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
SubTp);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 887802c950982..4d9ff8d2d767f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -249,7 +249,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index d8ba99a4da023..d9b83ea76a20e 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1224,13 +1224,11 @@ InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
return NumOps;
}
-InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost ARMTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 3e6ad7f13b4e7..0ad782320ecc0 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -195,7 +195,9 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool preferInLoopReduction(RecurKind Kind, Type *Ty) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 466b5ab3a0aed..fad70ff87cc75 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -231,12 +231,11 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
-InstructionCost
-HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost HexagonTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
return 1;
}
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 1189b9bc9e065..59642d6cf2d97 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -128,7 +128,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 3b2c5b06e0cfa..9d46853e5d583 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -617,13 +617,11 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(
return Cost * CostFactor;
}
-InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost PPCTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
InstructionCost CostFactor =
vectorCostAdjustmentFactor(Instruction::ShuffleVector, SrcTy, nullptr);
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index b8e7277f2ff51..a16463275cb1e 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -115,7 +115,9 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 43b4791998d39..54ffa1504823a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -733,12 +733,11 @@ InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
return FirstSlideCost + SecondSlideCost + MaskCost;
}
-InstructionCost
-RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost RISCVTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
@@ -746,6 +745,9 @@ RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
"Expected the same scalar types");
Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
+ if (VIC == TTI::VectorInstrContext::SplatOpFolded &&
+ ST->sinkSplatOperands() && Kind == TTI::SK_Broadcast)
+ return TTI::TCC_Free;
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
// For now, skip all fixed vector cost analysis when P extension is available
@@ -2650,6 +2652,13 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(
return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
VIC);
+ // Scalar splat operand can be folded for vector ops that support splatting
+ // the scalar operand, so the explicit insertelement is free in this context.
+ if (Opcode == Instruction::InsertElement &&
+ VIC == TTI::VectorInstrContext::SplatOpFolded &&
+ ST->sinkSplatOperands() && Index == 0)
+ return TTI::TCC_Free;
+
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
@@ -3622,6 +3631,33 @@ bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
}
}
+TargetTransformInfo::VectorInstrContext RISCVTTIImpl::getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
+ GatherUseOps) const {
+ if (Scalars.empty() || !ST->hasVInstructions() || !ST->sinkSplatOperands() ||
+ !ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))
+ return VectorInstrContext::None;
+
+ const auto *SplatIt = find_if_not(Scalars, IsaPred<UndefValue>);
+ if (SplatIt == Scalars.end() || (*SplatIt)->getType()->isIntegerTy(1) ||
+ isa<VectorType>((*SplatIt)->getType()) ||
+ isa<ExtractElementInst>(*SplatIt))
+ return VectorInstrContext::None;
+
+ SmallVector<TargetTransformInfo::BuildVectorUseOp, 4> UserOps;
+ if (!GatherUseOps(UserOps) || UserOps.empty())
+ return VectorInstrContext::None;
+
+ if (all_of(UserOps,
+ [this](const TargetTransformInfo::BuildVectorUseOp &UserOp) {
+ return canSplatOperand(UserOp.Opcode, UserOp.OperandIndex);
+ }))
+ return VectorInstrContext::SplatOpFolded;
+
+ return VectorInstrContext::None;
+}
+
/// Check if sinking \p I's operands to I's basic block is profitable, because
/// the operands can be folded into a target instruction, e.g.
/// splats of scalars can fold into vector instructions.
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 4523ebd6ae892..d6e904c53dc70 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -178,7 +178,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
@@ -546,6 +548,12 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
/// able to splat the given operand.
bool canSplatOperand(unsigned Opcode, int Operand) const;
+ TargetTransformInfo::VectorInstrContext getBuildVectorContextHint(
+ ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
+ function_ref<
+ bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
+ GatherUseOps) const override;
+
bool isProfitableToSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 8d6dee7c03361..e3c0aa2755175 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -794,12 +794,11 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
Args, CxtI);
}
-InstructionCost
-SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
- VectorType *SrcTy, TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost SystemZTTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
if (ST->hasVector()) {
unsigned NumVectors = getNumVectorRegs(SrcTy);
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index e022414014352..36a2bc90bdcbb 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -118,7 +118,9 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const;
unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const;
unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index d790dfae1d0be..4b392a7c802dc 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -302,8 +302,8 @@ InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
InstructionCost WebAssemblyTTIImpl::getShuffleCost(
TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
- VectorType *SubTp, ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
// Canonicalize the ShuffleKind in case optimizations didn't.
// Otherwise, we might end up with the wrong ShuffleKind to match against.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 957813ab54d79..83c1736fcee4d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -90,7 +90,9 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
using BaseT::getVectorInstrCost;
InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e41870a4ecbfa..3ca1fff52b449 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1945,13 +1945,11 @@ X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
return InstructionCost::getInvalid();
}
-InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
- VectorType *DstTy, VectorType *SrcTy,
- TTI::TargetCostKind CostKind,
- ArrayRef<int> Mask, int Index,
- VectorType *SubTp,
- ArrayRef<const Value *> Args,
- const Instruction *CxtI) const {
+InstructionCost X86TTIImpl::getShuffleCost(
+ TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
+ TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
+ VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CxtI,
+ TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index dd413277cdc86..f4197c260ba81 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -81,7 +81,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
- const Instruction *CxtI = nullptr) const override;
+ const Instruction *CxtI = nullptr,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a839118c31a3..2a2b6bc1c3a03 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14684,7 +14684,22 @@ void BoUpSLP::transformNodes() {
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
bool IsFinalized = false;
SmallVector<int> CommonMask;
- SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
+
+ // Each input vector can be tagged with extra context information
+ struct InVector {
+ PointerUnion<Value *, const TreeEntry *> Vec = nullptr;
+ TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None;
+
+ InVector(Value *Vec, TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None)
+ : Vec(Vec), Ctx(Ctx) {}
+ InVector(const TreeEntry *Vec,
+ TargetTransformInfo::VectorInstrContext Ctx =
+ TargetTransformInfo::VectorInstrContext::None)
+ : Vec(Vec), Ctx(Ctx) {}
+ };
+ SmallVector<InVector, 2> InVectors;
/// Captures the original scalar VL of a single, "clean" gather() call so
/// the values can be forwarded as the Args operand to getShuffleCost() for
/// the final permutation in finalize(). This lets the target cost model
@@ -14724,7 +14739,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return Constant::getAllOnesValue(Ty);
}
- InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root) {
+ InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root,
+ TTI::VectorInstrContext ContextHint) {
if ((!Root && allConstant(VL)) || all_of(VL, IsaPred<UndefValue>))
return TTI::TCC_Free;
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
@@ -14748,7 +14764,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
CostKind, std::distance(VL.begin(), It),
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It,
+ ContextHint);
}
SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
@@ -14757,12 +14774,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
});
InstructionCost InsertCost =
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
- PoisonValue::get(VecTy), *It);
+ PoisonValue::get(VecTy), *It, ContextHint);
return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast,
VecTy, CostKind, ShuffleMask,
/*Index=*/0, /*SubTp=*/nullptr,
- /*Args=*/*It);
+ /*Args=*/*It, ContextHint);
}
+ assert(ContextHint != TTI::VectorInstrContext::SplatOpFolded &&
+ "Expected SplatOpFolded to be a splat");
return GatherCost + (all_of(VL, IsaPred<UndefValue>)
? TTI::TCC_Free
: R.getGatherCost(VL, !Root, ScalarTy));
@@ -14913,9 +14932,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// sub-Mask into the CommonMask to estimate it later and avoid double cost
// estimation.
if ((InVectors.size() == 2 &&
- cast<const TreeEntry *>(InVectors.front()) == &E1 &&
- cast<const TreeEntry *>(InVectors.back()) == E2) ||
- (!E2 && cast<const TreeEntry *>(InVectors.front()) == &E1)) {
+ cast<const TreeEntry *>(InVectors.front().Vec) == &E1 &&
+ cast<const TreeEntry *>(InVectors.back().Vec) == E2) ||
+ (!E2 && cast<const TreeEntry *>(InVectors.front().Vec) == &E1)) {
unsigned Limit = getNumElems(Mask.size(), SliceSize, Part);
assert(all_of(ArrayRef(CommonMask).slice(Part * SliceSize, Limit),
[](int Idx) { return Idx == PoisonMaskElem; }) &&
@@ -14926,43 +14945,51 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
// Found non-matching nodes - need to estimate the cost for the matched
// and transform mask.
- Cost += createShuffle(InVectors.front(),
- InVectors.size() == 1 ? nullptr : InVectors.back(),
- CommonMask);
+ if (InVectors.size() == 1)
+ Cost += createShuffle(InVectors.front().Vec, nullptr, CommonMask, {},
+ InVectors.front().Ctx);
+ else
+ Cost += createShuffle(InVectors.front().Vec, InVectors.back().Vec,
+ CommonMask, {}, InVectors.front().Ctx,
+ InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else if (InVectors.size() == 2) {
- Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask);
+ Cost +=
+ createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
+ {}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
SameNodesEstimated = false;
if (!E2 && InVectors.size() == 1) {
unsigned VF = E1.getVectorFactor();
- if (Value *V1 = dyn_cast<Value *>(InVectors.front())) {
+ if (Value *V1 = InVectors.front().Vec.dyn_cast<Value *>()) {
VF = std::max(VF, getVF(V1));
} else {
- const auto *E = cast<const TreeEntry *>(InVectors.front());
+ const auto *E = cast<const TreeEntry *>(InVectors.front().Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
- Cost += createShuffle(InVectors.front(), &E1, CommonMask);
+ Cost += createShuffle(InVectors.front().Vec, &E1, CommonMask, {},
+ InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else {
auto P = InVectors.front();
- Cost += createShuffle(&E1, E2, Mask);
+ Cost += createShuffle(&E1, E2, Mask, {}, TTI::VectorInstrContext::None);
unsigned VF = Mask.size();
- if (Value *V1 = dyn_cast<Value *>(P)) {
+ if (Value *V1 = dyn_cast<Value *>(P.Vec)) {
VF = std::max(VF,
getNumElements(V1->getType()));
} else {
- const auto *E = cast<const TreeEntry *>(P);
+ const auto *E = cast<const TreeEntry *>(P.Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx + (InVectors.empty() ? 0 : VF);
- Cost += createShuffle(P, InVectors.front(), CommonMask);
+ Cost += createShuffle(P.Vec, InVectors.front().Vec, CommonMask, {}, P.Ctx,
+ InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
}
@@ -14995,8 +15022,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(V1->getType()), CostKind, Mask);
}
- InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask,
- ArrayRef<Value *> VL) const {
+ InstructionCost
+ createShuffleVector(Value *V1, ArrayRef<int> Mask, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext ContextHint =
+ TTI::VectorInstrContext::None) const {
// Empty mask or identity mask are free.
unsigned VF =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
@@ -15004,7 +15033,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
return TTI::TCC_Free;
return getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(V1->getType()), CostKind, Mask,
- /*Index=*/0, /*SubTp=*/nullptr, VL);
+ /*Index=*/0, /*SubTp=*/nullptr, VL, ContextHint);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
@@ -15019,7 +15048,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
InstructionCost
createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
const PointerUnion<Value *, const TreeEntry *> &P2,
- ArrayRef<int> Mask, ArrayRef<Value *> VL = {}) {
+ ArrayRef<int> Mask, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext Ctx1 = TTI::VectorInstrContext::None,
+ TTI::VectorInstrContext Ctx2 = TTI::VectorInstrContext::None) {
ShuffleCostBuilder Builder(TTI, CostKind);
SmallVector<int> CommonMask(Mask);
Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
@@ -15226,8 +15257,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
if (InVectors.size() == 2)
InVectors.pop_back();
- return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
- V1, V2, CommonMask, Builder, ScalarTy, SLPReVec, VL);
+ TTI::VectorInstrContext Ctx =
+ P2.isNull() ? Ctx1 : TTI::combineVectorInstrContexts(Ctx1, Ctx2);
+ return ExtraCost +
+ BaseShuffleAnalysis::createShuffle<InstructionCost>(
+ V1, V2, CommonMask, Builder, ScalarTy, SLPReVec, VL, Ctx);
}
public:
@@ -15429,7 +15463,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
if (P.value() == PoisonMaskElem)
return Mask[P.index()] == PoisonMaskElem;
auto *EI = cast<ExtractElementInst>(
- cast<const TreeEntry *>(InVectors.front())
+ cast<const TreeEntry *>(InVectors.front().Vec)
->getOrdered(P.index()));
return EI->getVectorOperand() == V1 ||
EI->getVectorOperand() == V2;
@@ -15437,23 +15471,25 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
"Expected extractelement vectors.");
}
/// Adds another one input vector and the mask for the shuffling.
- void add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false) {
+ void
+ add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false,
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
if (BVValues && !isa<Constant>(V1))
BVValues.reset();
if (InVectors.empty()) {
assert(CommonMask.empty() && !ForExtracts &&
"Expected empty input mask/vectors.");
CommonMask.assign(Mask.begin(), Mask.end());
- InVectors.assign(1, V1);
+ InVectors.assign(1, InVector(V1, ContextHint));
return;
}
if (ForExtracts) {
// No need to add vectors here, already handled them in adjustExtracts.
- assert(InVectors.size() == 1 && isa<const TreeEntry *>(InVectors[0]) &&
- !CommonMask.empty() &&
+ assert(InVectors.size() == 1 &&
+ isa<const TreeEntry *>(InVectors[0].Vec) && !CommonMask.empty() &&
all_of(enumerate(CommonMask),
[&](auto P) {
- Value *Scalar = cast<const TreeEntry *>(InVectors[0])
+ Value *Scalar = cast<const TreeEntry *>(InVectors[0].Vec)
->getOrdered(P.index());
if (P.value() == PoisonMaskElem)
return P.value() == Mask[P.index()] ||
@@ -15470,25 +15506,28 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
"Expected only tree entries from extracts/reused buildvectors.");
unsigned VF = getVF(V1);
if (InVectors.size() == 2) {
- Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask);
+ Cost +=
+ createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
+ {}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
VF = std::max<unsigned>(VF, CommonMask.size());
} else if (const auto *InTE =
- InVectors.front().dyn_cast<const TreeEntry *>()) {
+ InVectors.front().Vec.dyn_cast<const TreeEntry *>()) {
VF = std::max(VF, InTE->getVectorFactor());
} else {
- VF = std::max(
- VF, cast<FixedVectorType>(cast<Value *>(InVectors.front())->getType())
- ->getNumElements());
+ VF = std::max(VF, cast<FixedVectorType>(
+ cast<Value *>(InVectors.front().Vec)->getType())
+ ->getNumElements());
}
- InVectors.push_back(V1);
+ InVectors.emplace_back(V1, ContextHint);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
}
- Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
- Value *Root = nullptr) {
- Cost += getBuildVectorCost(VL, Root);
+ Value *
+ gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, Value *Root = nullptr,
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
+ Cost += getBuildVectorCost(VL, Root, ContextHint);
if (BVValues) {
if (BVValues->empty() && InVectors.empty())
BVValues->assign(VL.begin(), VL.end());
@@ -15538,11 +15577,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Action = {}) {
IsFinalized = true;
if (Action) {
- const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
+ const PointerUnion<Value *, const TreeEntry *> &Vec =
+ InVectors.front().Vec;
if (InVectors.size() == 2)
- Cost += createShuffle(Vec, InVectors.back(), CommonMask);
+ Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
+ InVectors.front().Ctx, InVectors.back().Ctx);
else
- Cost += createShuffle(Vec, nullptr, CommonMask);
+ Cost +=
+ createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
assert(VF > 0 &&
"Expected vector length for the final value before action.");
@@ -15554,11 +15596,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
InVectors.front() = V;
}
if (!SubVectors.empty()) {
- const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
+ const PointerUnion<Value *, const TreeEntry *> &Vec =
+ InVectors.front().Vec;
if (InVectors.size() == 2)
- Cost += createShuffle(Vec, InVectors.back(), CommonMask);
+ Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
+ InVectors.front().Ctx, InVectors.back().Ctx);
else
- Cost += createShuffle(Vec, nullptr, CommonMask);
+ Cost +=
+ createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
// Add subvectors permutation cost.
if (!SubVectorsMask.empty()) {
@@ -15629,10 +15674,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
ArrayRef<Value *> VL;
if (BVValues)
VL = *BVValues;
- return Cost +
- createShuffle(InVectors.front(),
- InVectors.size() == 2 ? InVectors.back() : nullptr,
- CommonMask, VL);
+ if (InVectors.size() == 2)
+ return Cost + createShuffle(InVectors.front().Vec, InVectors.back().Vec,
+ CommonMask, VL, InVectors.front().Ctx,
+ InVectors.back().Ctx);
+ return Cost + createShuffle(InVectors.front().Vec, nullptr, CommonMask, VL,
+ InVectors.front().Ctx);
}
~ShuffleCostEstimator() {
@@ -22210,7 +22257,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
InVectors.push_back(V1);
}
/// Adds another one input vector and the mask for the shuffling.
- void add(Value *V1, ArrayRef<int> Mask, bool = false) {
+ void add(Value *V1, ArrayRef<int> Mask, bool = false,
+ TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
assert(isa<FixedVectorType>(V1->getType()) &&
"castToScalarTyElem expects V1 to be FixedVectorType");
V1 = castToScalarTyElem(V1);
@@ -22269,7 +22317,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
add(V1, NewMask);
}
Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
- Value *Root = nullptr) {
+ Value *Root = nullptr,
+ TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
return R.gather(VL, Root, ScalarTy,
[&](Value *V1, Value *V2, ArrayRef<int> Mask) {
return createShuffle(V1, V2, Mask);
@@ -22955,10 +23004,39 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
- Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size());
- ShuffleBuilder.add(BV, ReuseMask);
+ TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None;
+ if constexpr (std::is_same_v<ResTy, InstructionCost>) {
+ auto GatherUserOps =
+ [&](SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) {
+ UserOps.clear();
+ if (NeedFreeze)
+ return false;
+ for (const auto &TE : VectorizableTree) {
+ if (DeletedNodes.contains(TE.get()))
+ continue;
+ if (!(TE->isGather() ||
+ TransformedToGatherNodes.contains(TE.get())) ||
+ !E->isSame(TE->Scalars))
+ continue;
+ auto *UserTE = TE->UserTreeIndex.UserTE;
+ if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle() ||
+ TransformedToGatherNodes.contains(UserTE))
+ return false;
+ UserOps.emplace_back(UserTE->getOpcode(),
+ TE->UserTreeIndex.EdgeIdx);
+ }
+ assert(UserOps.size() &&
+ "Ought to at least match with current entry");
+ return true;
+ };
+ ContextHint =
+ TTI->getBuildVectorContextHint(ReuseMask, E->Scalars, GatherUserOps);
+ }
+ Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(),
+ /*Root*/ nullptr, ContextHint);
+ ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
- SubVectorsMask);
+ SubVectorsMask, /*VF*/ 0, /*Action*/ {});
} else {
// Gather all constants.
SmallVector<int> Mask(GatheredScalars.size(), PoisonMaskElem);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
index a8259d158cf19..71cde2765d02b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
@@ -38,14 +38,15 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
TTI::ShuffleKind Kind, VectorType *Tp,
const TTI::TargetCostKind CostKind,
ArrayRef<int> Mask, int Index, VectorType *SubTp,
- ArrayRef<const Value *> Args) {
+ ArrayRef<const Value *> Args,
+ TTI::VectorInstrContext VIC) {
VectorType *DstTy = Tp;
if (!Mask.empty())
DstTy = FixedVectorType::get(Tp->getScalarType(), Mask.size());
if (Kind != TTI::SK_PermuteTwoSrc)
return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp,
- Args);
+ Args, /*CxtI=*/nullptr, VIC);
int NumSrcElts = Tp->getElementCount().getKnownMinValue();
int NumSubElts;
if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
@@ -55,8 +56,8 @@ InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
return TTI.getShuffleCost(TTI::SK_InsertSubvector, DstTy, Tp, CostKind,
Mask, Index, Tp);
}
- return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp,
- Args);
+ return TTI.getShuffleCost(Kind, DstTy, Tp, CostKind, Mask, Index, SubTp, Args,
+ /*CxtI=*/nullptr, VIC);
}
std::pair<InstructionCost, InstructionCost>
@@ -214,8 +215,8 @@ getScalarizationOverhead(const TargetTransformInfo &TTI, bool ReVec,
InstructionCost getVectorInstrCost(
const TargetTransformInfo &TTI, bool ReVec, Type *ScalarTy, unsigned Opcode,
Type *Val, const TTI::TargetCostKind CostKind, unsigned Index,
- Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) {
+ Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC) {
if (Opcode == Instruction::ExtractElement) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(ReVec && "Only supported by REVEC.");
@@ -226,7 +227,7 @@ InstructionCost getVectorInstrCost(
}
}
return TTI.getVectorInstrCost(Opcode, Val, CostKind, Index, Scalar,
- ScalarUserAndIdx);
+ ScalarUserAndIdx, VIC);
}
InstructionCost getExtractWithExtendCost(const TargetTransformInfo &TTI,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
index a2956043070de..321ff5b087229 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.h
@@ -47,7 +47,9 @@ getShuffleCost(const TargetTransformInfo &TTI,
TargetTransformInfo::ShuffleKind Kind, VectorType *Tp,
const TargetTransformInfo::TargetCostKind CostKind,
ArrayRef<int> Mask = {}, int Index = 0,
- VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {});
+ VectorType *SubTp = nullptr, ArrayRef<const Value *> Args = {},
+ TargetTransformInfo::VectorInstrContext VIC =
+ TargetTransformInfo::VectorInstrContext::None);
/// Calculate the scalar and the vector costs from vectorizing set of GEPs.
std::pair<InstructionCost, InstructionCost>
@@ -112,7 +114,8 @@ getVectorInstrCost(const TargetTransformInfo &TTI, bool ReVec, Type *ScalarTy,
unsigned Opcode, Type *Val,
const TargetTransformInfo::TargetCostKind CostKind,
unsigned Index, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx);
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None);
/// This is similar to TargetTransformInfo::getExtractWithExtendCost, but if Dst
/// is a FixedVectorType, a vector will be extracted instead of a scalar.
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
index 8f738c3978029..b5dbe1b089df8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/check-node-without-vector-user.ll
@@ -6,37 +6,17 @@
define void @test(i64 %0, ptr %1) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr @r, align 1
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i8 [[TMP3]] to i1
-; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr @r, i64 [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule i8 [[TMP3]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = sext i1 [[TMP8]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -8049), align 1
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i8 [[TMP10]] to i1
-; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr @r, i64 [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ule i8 [[TMP10]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = sext i1 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP17:%.*]] = add i32 [[TMP9]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -16098), align 1
-; CHECK-NEXT: [[TMP19:%.*]] = trunc i8 [[TMP18]] to i1
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr @r, i64 [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP21]], align 1
-; CHECK-NEXT: [[TMP23:%.*]] = icmp ule i8 [[TMP18]], [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = sext i1 [[TMP23]] to i32
-; CHECK-NEXT: [[TMP25:%.*]] = add i32 [[TMP17]], [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr getelementptr (i8, ptr @r, i64 -24147), align 1
-; CHECK-NEXT: [[TMP27:%.*]] = trunc i8 [[TMP26]] to i1
-; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr @r, i64 [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = load i8, ptr [[TMP29]], align 1
-; CHECK-NEXT: [[TMP31:%.*]] = icmp ule i8 [[TMP26]], [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = sext i1 [[TMP31]] to i32
-; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[TMP25]], [[TMP32]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i8> @llvm.experimental.vp.strided.load.v4i8.p0.i64(ptr align 1 getelementptr (i8, ptr @r, i64 -24147), i64 8049, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> <ptr @r, ptr @r, ptr @r, ptr @r>, <4 x i64> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP8]], <4 x i1> splat (i1 true), <4 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ule <4 x i8> [[TMP3]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i8>
+; CHECK-NEXT: [[TMP12:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP11]])
+; CHECK-NEXT: [[TMP33:%.*]] = sext i8 [[TMP12]] to i32
; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP1]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
index a0d3d2b136833..861ec175a6585 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/folded-broadcast-cost.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: fmul_splat
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-7'
+; YAML-NEXT: - Cost: '-9'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '4'
; YAML-NEXT: ...
@@ -18,7 +18,7 @@
; YAML-NEXT: Function: fmul_splat_nested
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-13'
+; YAML-NEXT: - Cost: '-15'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '8'
; YAML-NEXT: ...
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
index 5ef40a3f6d2f7..8c640da2fa718 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remark-zext-incoming-for-neg-icmp.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: test
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost '
-; YAML-NEXT: - Cost: '-10'
+; YAML-NEXT: - Cost: '-12'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '8'
; YAML-NEXT:...
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
index 9a9fcd1b259c3..6501394690d97 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-buildvector-scalars.ll
@@ -54,10 +54,10 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-SAME: i32 [[TMP0:%.*]], i32 [[ADD111_I_I:%.*]], <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT:%.*]], <4 x i32> [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; THRESH-NEXT: [[ENTRY:.*:]]
; THRESH-NEXT: [[MUL1445_I:%.*]] = shl i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2235_I16:%.*]] = or i32 [[TMP0]], 1
-; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
; THRESH-NEXT: [[TMP31:%.*]] = shufflevector <2 x i32> [[TMP30]], <2 x i32> poison, <2 x i32> zeroinitializer
+; THRESH-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP31]], <i32 1, i32 0>
+; THRESH-NEXT: [[ADD2323_I:%.*]] = add i32 [[TMP0]], 1
; THRESH-NEXT: [[TMP32:%.*]] = add <2 x i32> [[TMP31]], <i32 0, i32 1>
; THRESH-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP32]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
; THRESH-NEXT: [[TMP21:%.*]] = ashr <4 x i32> [[TMP5]], <i32 1, i32 0, i32 0, i32 1>
@@ -71,8 +71,8 @@ define fastcc i32 @test(i32 %0, i32 %add111.i.i, <4 x i32> %PredPel.i.sroa.86.72
; THRESH-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[PREDPEL_I_SROA_86_72_VEC_EXTRACT]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 poison, i32 poison, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <3 x i32> [[TMP4]], <3 x i32> poison, <8 x i32> <i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 0, i32 poison>
; THRESH-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 poison, i32 10, i32 3, i32 poison, i32 poison, i32 14, i32 poison>
-; THRESH-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP0]], i64 4
-; THRESH-NEXT: [[TMP36:%.*]] = insertelement <8 x i32> [[TMP35]], i32 [[ADD2235_I16]], i64 1
+; THRESH-NEXT: [[TMP35:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; THRESH-NEXT: [[TMP36:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP35]], <8 x i32> <i32 0, i32 8, i32 2, i32 3, i32 11, i32 5, i32 6, i32 7>
; THRESH-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[MUL1445_I]], i64 5
; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
>From 80df7ee872c0584459c8571fed35ccd2a0e73ba1 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 18 Sep 2026 13:44:31 -0700
Subject: [PATCH 2/3] [SLP][RISCV] Add new test to expose assertion failure
---
.../SLPVectorizer/RISCV/reordered-gather.ll | 38 +++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
new file mode 100644
index 0000000000000..3d2323dd91b5d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=slp-vectorizer -S -mtriple=riscv64 -mattr=+v -slp-threshold=-100 < %s | FileCheck %s
+
+; The gather will be reordered as it can decrease code size. This causes
+; an assertion failure in #213104 because E.isSame(E.Scalars) == false.
+
+define void @reordered_gather(ptr %in, ptr %out, i64 %a, i64 %b, i64 %c) #0 {
+entry:
+ %p0 = getelementptr i64, ptr %in, i64 0
+ %p1 = getelementptr i64, ptr %in, i64 1
+ %p2 = getelementptr i64, ptr %in, i64 2
+ %p3 = getelementptr i64, ptr %in, i64 3
+ %v0 = load i64, ptr %p0, align 4
+ %v1 = load i64, ptr %p1, align 4
+ %v2 = load i64, ptr %p2, align 4
+ %v3 = load i64, ptr %p3, align 4
+
+ %g = ptrtoint ptr %in to i64
+ %h = ptrtoint ptr %in to i64
+
+ %s0 = select i1 0, i64 0, i64 0
+ %s1 = select i1 0, i64 0, i64 0
+ %a0 = add i64 %v0, %g
+ %a1 = add i64 %v1, %s0
+ %a2 = add i64 %v2, %g
+ %a3 = add i64 %v3, %s1
+ %o0 = getelementptr i64, ptr %out, i64 0
+ %o1 = getelementptr i64, ptr %out, i64 1
+ %o2 = getelementptr i64, ptr %out, i64 2
+ %o3 = getelementptr i64, ptr %out, i64 3
+ store i64 %a0, ptr %o0, align 4
+ store i64 %a1, ptr %o1, align 4
+ store i64 %a2, ptr %o2, align 4
+ store i64 %a3, ptr %o3, align 4
+ ret void
+}
+
+attributes #0 = { optsize }
>From fc50c26384ff2ae3f87728ab83c8e054b82d0860 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 18 Sep 2026 13:48:45 -0700
Subject: [PATCH 3/3] [SLP] Remove assertion require UserOps to be populated
Gathers may not match themsevles if they are reordered
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +---
.../SLPVectorizer/RISCV/reordered-gather.ll | 15 +++++++++++++++
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2a2b6bc1c3a03..56efc39b47a4e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23025,9 +23025,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
UserOps.emplace_back(UserTE->getOpcode(),
TE->UserTreeIndex.EdgeIdx);
}
- assert(UserOps.size() &&
- "Ought to at least match with current entry");
- return true;
+ return !UserOps.empty();
};
ContextHint =
TTI->getBuildVectorContextHint(ReuseMask, E->Scalars, GatherUserOps);
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
index 3d2323dd91b5d..418438e2082af 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reordered-gather.ll
@@ -5,6 +5,21 @@
; an assertion failure in #213104 because E.isSame(E.Scalars) == false.
define void @reordered_gather(ptr %in, ptr %out, i64 %a, i64 %b, i64 %c) #0 {
+; CHECK-LABEL: define void @reordered_gather(
+; CHECK-SAME: ptr [[IN:%.*]], ptr [[OUT:%.*]], i64 [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i64, ptr [[IN]], i64 0
+; CHECK-NEXT: [[V0:%.*]] = ptrtoint ptr [[IN]] to i64
+; CHECK-NEXT: [[H:%.*]] = ptrtoint ptr [[IN]] to i64
+; CHECK-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[OUT]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[P0]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[V0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> <i64 undef, i64 undef, i64 0, i64 0>, <4 x i64> [[TMP2]], <4 x i32> <i32 4, i32 2, i32 6, i32 3>
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP3]], [[TMP7]]
+; CHECK-NEXT: store <4 x i64> [[TMP8]], ptr [[O0]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%p0 = getelementptr i64, ptr %in, i64 0
%p1 = getelementptr i64, ptr %in, i64 1
More information about the llvm-commits
mailing list