[llvm] [AArch64] Use VectorInstrContext in getScalarizationOverhead. (PR #177201)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 21 08:51:46 PST 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/177201
Use VectorInstrContext to return more accurate scalarization overhead
costs when inserts/extracts can be folded into ld1/st1 and CPUs where
ld1/st1 are fast (same perf as regular loads).
Depends on https://github.com/llvm/llvm-project/pull/175982 (included in
PR).
>From e0a2136f8f7646fb33aff522af15a175ade3795a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 14 Jan 2026 16:19:00 +0000
Subject: [PATCH 1/2] [TTI] Add VectorInstrContext for context-aware
insert/extract costs.
This commit introduces the VectorInstrContext (VIC) infrastructure to
improve cost estimates for insert/extracts based on the context
instruction in which the insert/extract is used.
This is similar to CastContextHint, and allows providing context on how
the insert/extract is going to be used before creating IR. This is
useful in the LoopVectorizer, where costs need to estimated before
creating IR.
The new hint currently only replaces an existing check in AArch64, but
I plan to add additional uses of the store context hint as a follow-up
---
.../llvm/Analysis/TargetTransformInfo.h | 44 ++++++++---
.../llvm/Analysis/TargetTransformInfoImpl.h | 32 ++++----
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 73 ++++++++++++-------
llvm/lib/Analysis/TargetTransformInfo.cpp | 48 ++++++++----
.../AArch64/AArch64TargetTransformInfo.cpp | 37 +++++-----
.../AArch64/AArch64TargetTransformInfo.h | 42 ++++++-----
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 14 ++--
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 9 ++-
.../Target/AMDGPU/R600TargetTransformInfo.cpp | 15 ++--
.../Target/AMDGPU/R600TargetTransformInfo.h | 9 ++-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 13 ++--
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 9 ++-
.../Hexagon/HexagonTargetTransformInfo.cpp | 10 +--
.../Hexagon/HexagonTargetTransformInfo.h | 9 ++-
.../Target/NVPTX/NVPTXTargetTransformInfo.h | 11 ++-
.../Target/PowerPC/PPCTargetTransformInfo.cpp | 14 ++--
.../Target/PowerPC/PPCTargetTransformInfo.h | 9 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 15 ++--
.../Target/RISCV/RISCVTargetTransformInfo.h | 20 +++--
.../SystemZ/SystemZTargetTransformInfo.cpp | 14 ++--
.../SystemZ/SystemZTargetTransformInfo.h | 20 +++--
.../WebAssemblyTargetTransformInfo.cpp | 4 +-
.../WebAssemblyTargetTransformInfo.h | 9 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 23 +++---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 20 +++--
.../Transforms/Vectorize/LoopVectorize.cpp | 16 +++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 14 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 2 +-
30 files changed, 337 insertions(+), 231 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 9db2e3977f71c..9d567467aa62d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1041,6 +1041,24 @@ class TargetTransformInfo {
isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
int RetIdx) const;
+ /// Represents a hint about the context in which an insert/extract is used.
+ ///
+ /// On some targets, inserts/extracts can cheaply be folded into loads/stores.
+ ///
+ /// This enum allows the vectorizer to give getVectorInstrCost an idea of how
+ /// inserts/extracts are used
+ ///
+ /// See \c getVectorInstrContextHint to compute a VectorInstrContext from an
+ /// insert/extract Instruction*.
+ enum class VectorInstrContext : uint8_t {
+ None, ///< The insert/extract is not used with a load/store.
+ Load, ///< The value being inserted comes from a load (InsertElement only).
+ Store, ///< The extracted value is stored (ExtractElement only).
+ };
+
+ /// Calculates a VectorInstrContext from \p I.
+ static VectorInstrContext getVectorInstrContextHint(const Instruction *I);
+
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
/// are set if the demanded result elements need to be inserted and/or
/// extracted from vectors. The involved values may be passed in VL if
@@ -1048,12 +1066,14 @@ class TargetTransformInfo {
LLVM_ABI InstructionCost getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const;
+ ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// Estimate the overhead of scalarizing operands with the given types. The
/// (potentially vector) types to use for each of argument are passes via Tys.
LLVM_ABI InstructionCost getOperandsScalarizationOverhead(
- ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const;
+ ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// If target has efficient vector element load/store instructions, it can
/// return true here so that insertion/extraction costs are not added to
@@ -1569,11 +1589,11 @@ class TargetTransformInfo {
/// This is used when the instruction is not available; a typical use
/// case is to provision the cost of vectorization/scalarization in
/// vectorizer passes.
- LLVM_ABI InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index = -1,
- const Value *Op0 = nullptr,
- const Value *Op1 = nullptr) const;
+ LLVM_ABI InstructionCost getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index = -1, const Value *Op0 = nullptr,
+ const Value *Op1 = nullptr,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// \return The expected cost of vector Insert and Extract.
/// Use -1 to indicate that there is no information on the index value.
@@ -1587,7 +1607,8 @@ class TargetTransformInfo {
LLVM_ABI InstructionCost getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const;
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// \return The expected cost of vector Insert and Extract.
/// This is used when instruction is available, and implementation
@@ -1595,9 +1616,10 @@ class TargetTransformInfo {
///
/// A typical suitable use case is cost estimation when vector instruction
/// exists (e.g., from basic blocks during transformation).
- LLVM_ABI InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index = -1) const;
+ LLVM_ABI InstructionCost getVectorInstrCost(
+ const Instruction &I, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index = -1,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
/// \return The expected cost of inserting or extracting a lane that is \p
/// Index elements from the end of a vector, i.e. the mathematical expression
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 07b3755924fd1..2a93b29930ad4 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -500,13 +500,16 @@ class TargetTransformInfoImplBase {
virtual InstructionCost getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const {
+ ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
+ // Default implementation returns 0.
+ // BasicTTIImpl provides the actual implementation.
return 0;
}
- virtual InstructionCost
- getOperandsScalarizationOverhead(ArrayRef<Type *> Tys,
- TTI::TargetCostKind CostKind) const {
+ virtual InstructionCost getOperandsScalarizationOverhead(
+ ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 0;
}
@@ -830,10 +833,10 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const {
+ virtual InstructionCost getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
@@ -844,13 +847,15 @@ class TargetTransformInfoImplBase {
virtual InstructionCost getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
- virtual InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index) const {
+ virtual InstructionCost getVectorInstrCost(
+ const Instruction &I, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
return 1;
}
@@ -1574,7 +1579,8 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
if (auto *CI = dyn_cast<ConstantInt>(Operands[2]))
if (CI->getValue().getActiveBits() <= 32)
Idx = CI->getZExtValue();
- return TargetTTI->getVectorInstrCost(*IE, Ty, CostKind, Idx);
+ return TargetTTI->getVectorInstrCost(*IE, Ty, CostKind, Idx,
+ TTI::getVectorInstrContextHint(IE));
}
case Instruction::ShuffleVector: {
auto *Shuffle = dyn_cast<ShuffleVectorInst>(U);
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index c430e11168f73..2fc48baf58323 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -380,6 +380,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
~BasicTTIImplBase() override = default;
using TargetTransformInfoImplBase::DL;
+ using TargetTransformInfoImplBase::getScalarizationOverhead;
public:
/// \name Scalar TTI Implementations
@@ -893,10 +894,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
/// are set if the demanded result elements need to be inserted and/or
/// extracted from vectors.
- InstructionCost getScalarizationOverhead(
- VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override {
+ InstructionCost
+ getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
/// FIXME: a bitfield is not a reasonable abstraction for talking about
/// which elements are needed from a scalable vector
if (isa<ScalableVectorType>(InTy))
@@ -914,12 +918,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
continue;
if (Insert) {
Value *InsertedVal = VL.empty() ? nullptr : VL[i];
- Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
- CostKind, i, nullptr, InsertedVal);
+ Cost +=
+ thisT()->getVectorInstrCost(Instruction::InsertElement, Ty,
+ CostKind, i, nullptr, InsertedVal, VIC);
}
if (Extract)
Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, Ty,
- CostKind, i, nullptr, nullptr);
+ CostKind, i, nullptr, nullptr, VIC);
}
return Cost;
@@ -947,23 +952,27 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}
/// Helper wrapper for the DemandedElts variant of getScalarizationOverhead.
- InstructionCost getScalarizationOverhead(VectorType *InTy, bool Insert,
- bool Extract,
- TTI::TargetCostKind CostKind) const {
+ InstructionCost getScalarizationOverhead(
+ VectorType *InTy, bool Insert, bool Extract, TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const {
if (isa<ScalableVectorType>(InTy))
return InstructionCost::getInvalid();
auto *Ty = cast<FixedVectorType>(InTy);
APInt DemandedElts = APInt::getAllOnes(Ty->getNumElements());
+ // Use CRTP to allow target overrides
return thisT()->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
- CostKind);
+ CostKind, ForPoisonSrc, VL, VIC);
}
/// Estimate the overhead of scalarizing an instruction's
/// operands. The (potentially vector) types to use for each of
/// argument are passes via Tys.
InstructionCost getOperandsScalarizationOverhead(
- ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const override {
+ ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
InstructionCost Cost = 0;
for (Type *Ty : Tys) {
// Disregard things like metadata arguments.
@@ -973,7 +982,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
if (auto *VecTy = dyn_cast<VectorType>(Ty))
Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
- /*Extract*/ true, CostKind);
+ /*Extract*/ true, CostKind,
+ /*ForPoisonSrc=*/true, {}, VIC);
}
return Cost;
@@ -1428,10 +1438,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return 1;
}
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override {
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
return getRegUsageForType(Val->getScalarType());
}
@@ -1439,26 +1450,32 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
/// vector with 'Scalar' being the value being extracted,'User' being the user
/// of the extract(nullptr if user is not known before vectorization) and
/// 'Idx' being the extract lane.
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>>
- ScalarUserAndIdx) const override {
- return thisT()->getVectorInstrCost(Opcode, Val, CostKind, Index, nullptr,
- nullptr);
+ InstructionCost getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ Value *Scalar,
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
+ return getVectorInstrCost(Opcode, Val, CostKind, Index, nullptr, nullptr,
+ VIC);
}
- InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index) const override {
+ InstructionCost
+ getVectorInstrCost(const Instruction &I, Type *Val,
+ TTI::TargetCostKind CostKind, unsigned Index,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
Value *Op0 = nullptr;
Value *Op1 = nullptr;
if (auto *IE = dyn_cast<InsertElementInst>(&I)) {
Op0 = IE->getOperand(0);
Op1 = IE->getOperand(1);
}
+ // If VIC is None, compute it from the instruction
+ if (VIC == TTI::VectorInstrContext::None)
+ VIC = TTI::getVectorInstrContextHint(&I);
return thisT()->getVectorInstrCost(I.getOpcode(), Val, CostKind, Index, Op0,
- Op1);
+ Op1, VIC);
}
InstructionCost
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index b2b77da4914d6..36ce3cfbdbacc 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -646,17 +646,35 @@ bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField(
return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
}
+TargetTransformInfo::VectorInstrContext
+TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) {
+ if (!I)
+ return VectorInstrContext::None;
+
+ // For inserts, check if the value being inserted comes from a load.
+ if (isa<InsertElementInst>(I) && isa<LoadInst>(I->getOperand(1)))
+ return VectorInstrContext::Load;
+
+ // For extracts, check if it has a single use that is a store.
+ if (isa<ExtractElementInst>(I) && I->hasOneUse() &&
+ isa<StoreInst>(*I->user_begin()))
+ return VectorInstrContext::Store;
+
+ return VectorInstrContext::None;
+}
+
InstructionCost TargetTransformInfo::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc,
- ArrayRef<Value *> VL) const {
+ TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext VIC) const {
return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
- CostKind, ForPoisonSrc, VL);
+ CostKind, ForPoisonSrc, VL, VIC);
}
InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
- ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const {
- return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind);
+ ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind,
+ TTI::VectorInstrContext VIC) const {
+ return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind, VIC);
}
bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
@@ -1124,37 +1142,37 @@ InstructionCost TargetTransformInfo::getCmpSelInstrCost(
InstructionCost TargetTransformInfo::getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
- const Value *Op0, const Value *Op1) const {
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
assert((Opcode == Instruction::InsertElement ||
Opcode == Instruction::ExtractElement) &&
"Expecting Opcode to be insertelement/extractelement.");
InstructionCost Cost =
- TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
+ TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
InstructionCost TargetTransformInfo::getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
- Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
+ Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC) const {
assert((Opcode == Instruction::InsertElement ||
Opcode == Instruction::ExtractElement) &&
"Expecting Opcode to be insertelement/extractelement.");
InstructionCost Cost = TTIImpl->getVectorInstrCost(
- Opcode, Val, CostKind, Index, Scalar, ScalarUserAndIdx);
+ Opcode, Val, CostKind, Index, Scalar, ScalarUserAndIdx, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
-InstructionCost
-TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index) const {
+InstructionCost TargetTransformInfo::getVectorInstrCost(
+ const Instruction &I, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, TTI::VectorInstrContext VIC) const {
// FIXME: Assert that Opcode is either InsertElement or ExtractElement.
// This is mentioned in the interface description and respected by all
// callers, but never asserted upon.
- InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index);
+ InstructionCost Cost =
+ TTIImpl->getVectorInstrCost(I, Val, CostKind, Index, VIC);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 0fec9e29a2d29..fade6f0bd17f7 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3986,7 +3986,8 @@ InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
const Instruction *I, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC) const {
assert(Val->isVectorTy() && "This must be a vector type");
if (Index != -1U) {
@@ -4015,7 +4016,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
// register instruction. I.e., if this is an `insertelement` instruction,
// and its second operand is a load, then we will generate a LD1, which
// are expensive instructions on some uArchs.
- if (I && isa<LoadInst>(I->getOperand(1))) {
+ if (VIC == TTI::VectorInstrContext::Load) {
if (ST->hasFastLD1Single())
return 0;
return CostKind == TTI::TCK_CodeSize
@@ -4156,33 +4157,33 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
: ST->getVectorInsertExtractBaseCost();
}
-InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index,
- const Value *Op0,
- const Value *Op1) const {
+InstructionCost AArch64TTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
// Treat insert at lane 0 into a poison vector as having zero cost. This
// ensures vector broadcasts via an insert + shuffle (and will be lowered to a
// single dup) are treated as cheap.
if (Opcode == Instruction::InsertElement && Index == 0 && Op0 &&
isa<PoisonValue>(Op0))
return 0;
- return getVectorInstrCostHelper(Opcode, Val, CostKind, Index);
+ return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, nullptr,
+ nullptr, {}, VIC);
}
InstructionCost AArch64TTIImpl::getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
- Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
+ Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC) const {
return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, nullptr, Scalar,
- ScalarUserAndIdx);
+ ScalarUserAndIdx, VIC);
}
-InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
- Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index) const {
- return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, &I);
+InstructionCost
+AArch64TTIImpl::getVectorInstrCost(const Instruction &I, Type *Val,
+ TTI::TargetCostKind CostKind, unsigned Index,
+ TTI::VectorInstrContext VIC) const {
+ return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, &I,
+ nullptr, {}, VIC);
}
InstructionCost
@@ -4205,8 +4206,8 @@ AArch64TTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
InstructionCost AArch64TTIImpl::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc,
- ArrayRef<Value *> VL) const {
+ TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext VIC) const {
if (isa<ScalableVectorType>(Ty))
return InstructionCost::getInvalid();
if (Ty->getElementType()->isFloatingPointTy())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index c9bf44b15144a..24f0848991c82 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -81,7 +81,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
InstructionCost getVectorInstrCostHelper(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
const Instruction *I = nullptr, Value *Scalar = nullptr,
- ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {}) const;
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx = {},
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None) const;
public:
explicit AArch64TTIImpl(const AArch64TargetMachine *TM, const Function &F)
@@ -214,24 +215,28 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
/// \param ScalarUserAndIdx encodes the information about extracts from a
/// vector with 'Scalar' being the value being extracted,'User' being the user
/// of the extract(nullptr if user is not known before vectorization) and
/// 'Idx' being the extract lane.
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, Value *Scalar,
- ArrayRef<std::tuple<Value *, User *, int>>
- ScalarUserAndIdx) const override;
+ InstructionCost getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ Value *Scalar,
+ ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
- InstructionCost getVectorInstrCost(const Instruction &I, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index) const override;
+ InstructionCost
+ getVectorInstrCost(const Instruction &I, Type *Val,
+ TTI::TargetCostKind CostKind, unsigned Index,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
@@ -500,10 +505,13 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost getScalarizationOverhead(
- VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override;
+ InstructionCost
+ getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
/// Return the cost of the scaling factor used in the addressing
/// mode represented by AM for this target, for a load/store
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index fbc0307263190..e0b8b329ab46c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -882,10 +882,9 @@ GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
return LT.first * getHalfRateInstrCost(CostKind);
}
-InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const {
+InstructionCost GCNTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
switch (Opcode) {
case Instruction::ExtractElement:
case Instruction::InsertElement: {
@@ -894,8 +893,8 @@ InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
if (EltSize < 32) {
if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
return 0;
- return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0,
- Op1);
+ return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC);
}
// Extracts are just reads of a subregister, so are free. Inserts are
@@ -906,7 +905,8 @@ InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
return Index == ~0u ? 2 : 0;
}
default:
- return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);
+ return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC);
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 4dcf381a9af93..3ec157aacd0aa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -176,10 +176,11 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
ArrayRef<unsigned> Indices = {}) const;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
index 3093227279a31..c08edc1bb5512 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.cpp
@@ -108,19 +108,17 @@ InstructionCost R600TTIImpl::getCFInstrCost(unsigned Opcode,
}
}
-InstructionCost R600TTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
- TTI::TargetCostKind CostKind,
- unsigned Index,
- const Value *Op0,
- const Value *Op1) const {
+InstructionCost R600TTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
switch (Opcode) {
case Instruction::ExtractElement:
case Instruction::InsertElement: {
unsigned EltSize =
DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
if (EltSize < 32) {
- return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0,
- Op1);
+ return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC);
}
// Extracts are just reads of a subregister, so are free. Inserts are
@@ -131,7 +129,8 @@ InstructionCost R600TTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
return Index == ~0u ? 2 : 0;
}
default:
- return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);
+ return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC);
}
}
diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
index 3deae69bfc8c9..ade1b1518215c 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -62,10 +62,11 @@ class R600TTIImpl final : public BasicTTIImplBase<R600TTIImpl> {
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
};
} // end namespace llvm
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index b947c8a10e2d8..94048360c0719 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -953,10 +953,9 @@ InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
}
-InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const {
+InstructionCost ARMTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
// Penalize inserting into an D-subregister. We end up with a three times
// lower estimated throughput on swift.
if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
@@ -975,7 +974,8 @@ InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
if (ValTy->isVectorTy() &&
ValTy->getScalarSizeInBits() <= 32)
return std::max<InstructionCost>(
- BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1),
+ BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC),
2U);
}
@@ -989,7 +989,8 @@ InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
}
- return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);
+ return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
+ VIC);
}
InstructionCost ARMTTIImpl::getCmpSelInstrCost(
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index fafd2d44a818c..94804152d96ec 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -354,10 +354,11 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getAddressComputationCost(Type *Val, ScalarEvolution *SE, const SCEV *Ptr,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 59c6201e07081..25ede7d262544 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -306,11 +306,9 @@ InstructionCost HexagonTTIImpl::getCastInstrCost(unsigned Opcode, Type *DstTy,
return 1;
}
-InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index,
- const Value *Op0,
- const Value *Op1) const {
+InstructionCost HexagonTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
Type *ElemTy = Val->isVectorTy() ? cast<VectorType>(Val)->getElementType()
: Val;
if (Opcode == Instruction::InsertElement) {
@@ -320,7 +318,7 @@ InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
return Cost;
// If it's not a 32-bit value, there will need to be an extract.
return Cost + getVectorInstrCost(Instruction::ExtractElement, Val, CostKind,
- Index, Op0, Op1);
+ Index, Op0, Op1, VIC);
}
if (Opcode == Instruction::ExtractElement)
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index edf88cf476f6d..0bd07a97ff3d5 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -145,10 +145,11 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index ae12a6ea3baa3..40eb161bc8666 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -119,10 +119,13 @@ class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost getScalarizationOverhead(
- VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override {
+ InstructionCost
+ getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override {
if (!InTy->getElementCount().isFixed())
return InstructionCost::getInvalid();
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 40e38a726cd9c..90d5128c75adc 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -685,10 +685,9 @@ InstructionCost PPCTTIImpl::getCmpSelInstrCost(
return Cost * CostFactor;
}
-InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const {
+InstructionCost PPCTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
assert(Val->isVectorTy() && "This must be a vector type");
int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -699,7 +698,7 @@ InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
return InstructionCost::getMax();
InstructionCost Cost =
- BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
+ BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC);
Cost *= CostFactor;
if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) {
@@ -862,8 +861,9 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
if (Src->isVectorTy() && Opcode == Instruction::Store)
for (int I = 0, E = cast<FixedVectorType>(Src)->getNumElements(); I < E;
++I)
- Cost += getVectorInstrCost(Instruction::ExtractElement, Src, CostKind, I,
- nullptr, nullptr);
+ Cost +=
+ getVectorInstrCost(Instruction::ExtractElement, Src, CostKind, I,
+ nullptr, nullptr, TTI::VectorInstrContext::None);
return Cost;
}
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 518edf06df2ea..18792d8f0e9cd 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -128,10 +128,11 @@ class PPCTTIImpl final : public BasicTTIImplBase<PPCTTIImpl> {
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost getMemoryOpCost(
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index c0119325285c7..20a4c5f039e12 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -984,8 +984,8 @@ static unsigned isM1OrSmaller(MVT VT) {
InstructionCost RISCVTTIImpl::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc,
- ArrayRef<Value *> VL) const {
+ TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext VIC) const {
if (isa<ScalableVectorType>(Ty))
return InstructionCost::getInvalid();
@@ -2410,11 +2410,9 @@ InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
return 0;
}
-InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index,
- const Value *Op0,
- const Value *Op1) const {
+InstructionCost RISCVTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
assert(Val->isVectorTy() && "This must be a vector type");
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
@@ -2426,7 +2424,8 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
if (Opcode != Instruction::ExtractElement &&
Opcode != Instruction::InsertElement)
- return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
+ return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
+ VIC);
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 6e38951520039..3434898d6d3e7 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -179,10 +179,13 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost getScalarizationOverhead(
- VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override;
+ InstructionCost
+ getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
@@ -246,10 +249,11 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 2611c291abaa6..4322773f4afd6 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -493,8 +493,8 @@ static bool isFreeEltLoad(const Value *Op) {
InstructionCost SystemZTTIImpl::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc,
- ArrayRef<Value *> VL) const {
+ TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext VIC) const {
unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
InstructionCost Cost = 0;
@@ -1181,11 +1181,9 @@ InstructionCost SystemZTTIImpl::getCmpSelInstrCost(
Op1Info, Op2Info);
}
-InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index,
- const Value *Op0,
- const Value *Op1) const {
+InstructionCost SystemZTTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
if (Opcode == Instruction::InsertElement) {
// Vector Element Load.
if (Op1 != nullptr && isFreeEltLoad(Op1))
@@ -1208,7 +1206,7 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
return Cost;
}
- return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
+ return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC);
}
// Check if a load may be folded as a memory operand in its user.
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index fc681dec1859a..f4ba29c987f09 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -85,10 +85,13 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
bool hasDivRemOp(Type *DataType, bool IsSigned) const override;
bool prefersVectorizedAddressing() const override { return false; }
bool LSRWithInstrQueries() const override { return true; }
- InstructionCost getScalarizationOverhead(
- VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override;
+ InstructionCost
+ getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool supportsEfficientVectorElementLoadStore() const override { return true; }
bool enableInterleavedAccessVectorization() const override { return true; }
@@ -118,10 +121,11 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
bool isFoldableLoad(const LoadInst *Ld,
const Instruction *&FoldedValue) const;
InstructionCost getMemoryOpCost(
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 5a0562cdd4a83..3c89f6c2fe761 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -383,9 +383,9 @@ InstructionCost WebAssemblyTTIImpl::getInterleavedMemoryOpCost(
InstructionCost WebAssemblyTTIImpl::getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
- const Value *Op0, const Value *Op1) const {
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
InstructionCost Cost = BasicTTIImplBase::getVectorInstrCost(
- Opcode, Val, CostKind, Index, Op0, Op1);
+ Opcode, Val, CostKind, Index, Op0, Op1, VIC);
// SIMD128's insert/extract currently only take constant indices.
if (Index == -1u)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index eba59c78da9b2..3bd2e6418aeaf 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -86,10 +86,11 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase<WebAssemblyTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond, bool UseMaskForGaps) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost getPartialReductionCost(
unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 608727b745925..f4d630976d1eb 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -4803,10 +4803,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
return BaseT::getIntrinsicInstrCost(ICA, CostKind);
}
-InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const {
+InstructionCost X86TTIImpl::getVectorInstrCost(
+ unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
+ const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
static const CostTblEntry SLMCostTbl[] = {
{ ISD::EXTRACT_VECTOR_ELT, MVT::i8, 4 },
{ ISD::EXTRACT_VECTOR_ELT, MVT::i16, 4 },
@@ -4948,14 +4947,15 @@ InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
return ShuffleCost + IntOrFpCost + RegisterFileMoveCost;
}
- return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1) +
+ return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
+ VIC) +
RegisterFileMoveCost;
}
InstructionCost X86TTIImpl::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc,
- ArrayRef<Value *> VL) const {
+ TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
+ TTI::VectorInstrContext VIC) const {
assert(DemandedElts.getBitWidth() ==
cast<FixedVectorType>(Ty)->getNumElements() &&
"Vector size mismatch");
@@ -4987,7 +4987,8 @@ InstructionCost X86TTIImpl::getScalarizationOverhead(
continue;
Cost += getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, I,
Constant::getNullValue(Ty),
- VL.empty() ? nullptr : VL[I]);
+ VL.empty() ? nullptr : VL[I],
+ TTI::VectorInstrContext::None);
}
return Cost;
}
@@ -5792,7 +5793,8 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
// Add the final extract element to the cost.
return ReductionCost + getVectorInstrCost(Instruction::ExtractElement, Ty,
- CostKind, 0, nullptr, nullptr);
+ CostKind, 0, nullptr, nullptr,
+ TTI::VectorInstrContext::None);
}
InstructionCost X86TTIImpl::getMinMaxCost(Intrinsic::ID IID, Type *Ty,
@@ -5971,7 +5973,8 @@ X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
// Add the final extract element to the cost.
return MinMaxCost + getVectorInstrCost(Instruction::ExtractElement, Ty,
- CostKind, 0, nullptr, nullptr);
+ CostKind, 0, nullptr, nullptr,
+ TTI::VectorInstrContext::None);
}
/// Calculate the cost of materializing a 64-bit value. This helper
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 4f672793a0fcf..b3dde1555d0a0 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -165,14 +165,18 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
- InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
- TTI::TargetCostKind CostKind,
- unsigned Index, const Value *Op0,
- const Value *Op1) const override;
- InstructionCost getScalarizationOverhead(
- VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
- TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
- ArrayRef<Value *> VL = {}) const override;
+ InstructionCost
+ getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+ unsigned Index, const Value *Op0, const Value *Op1,
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
+ InstructionCost
+ getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
+ TTI::VectorInstrContext VIC =
+ TTI::VectorInstrContext::None) const override;
InstructionCost
getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
const APInt &DemandedDstElts,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0811628fd56b3..c5198bbc1d88e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5650,11 +5650,17 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
if (!RetTy->isVoidTy() &&
(!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) {
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
+ if (isa<LoadInst>(I))
+ VIC = TTI::VectorInstrContext::Load;
+ else if (isa<StoreInst>(I))
+ VIC = TTI::VectorInstrContext::Store;
+
for (Type *VectorTy : getContainedTypes(RetTy)) {
Cost += TTI.getScalarizationOverhead(
cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getFixedValue()),
- /*Insert=*/true,
- /*Extract=*/false, CostKind);
+ /*Insert=*/true, /*Extract=*/false, CostKind,
+ /*ForPoisonSrc=*/true, {}, VIC);
}
}
@@ -5675,7 +5681,11 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
SmallVector<Type *> Tys;
for (auto *V : filterExtractingOperands(Ops, VF))
Tys.push_back(maybeVectorizeType(V->getType(), VF));
- return Cost + TTI.getOperandsScalarizationOverhead(Tys, CostKind);
+
+ TTI::VectorInstrContext OperandVIC = isa<StoreInst>(I)
+ ? TTI::VectorInstrContext::Store
+ : TTI::VectorInstrContext::None;
+ return Cost + TTI.getOperandsScalarizationOverhead(Tys, CostKind, OperandVIC);
}
void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a6a46e36b397d..68a5e183ff3f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1743,7 +1743,7 @@ VPCostContext::getOperandInfo(VPValue *V) const {
InstructionCost VPCostContext::getScalarizationOverhead(
Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
- bool AlwaysIncludeReplicatingR) {
+ TTI::VectorInstrContext VIC, bool AlwaysIncludeReplicatingR) {
if (VF.isScalar())
return 0;
@@ -1757,8 +1757,8 @@ InstructionCost VPCostContext::getScalarizationOverhead(
to_vector(getContainedTypes(toVectorizedTy(ResultTy, VF)))) {
ScalarizationCost += TTI.getScalarizationOverhead(
cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getFixedValue()),
- /*Insert=*/true,
- /*Extract=*/false, CostKind);
+ /*Insert=*/true, /*Extract=*/false, CostKind,
+ /*ForPoisonSrc=*/true, {}, VIC);
}
}
// Compute the cost of scalarizing the operands, skipping ones that do not
@@ -1776,5 +1776,5 @@ InstructionCost VPCostContext::getScalarizationOverhead(
Tys.push_back(toVectorizedTy(Types.inferScalarType(Op), VF));
}
return ScalarizationCost +
- TTI.getOperandsScalarizationOverhead(Tys, CostKind);
+ TTI.getOperandsScalarizationOverhead(Tys, CostKind, VIC);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 334bf9d0620ca..26e4d31696f8a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -375,12 +375,14 @@ struct VPCostContext {
/// Estimate the overhead of scalarizing a recipe with result type \p ResultTy
/// and \p Operands with \p VF. This is a convenience wrapper for the
- /// type-based getScalarizationOverhead API. If \p AlwaysIncludeReplicatingR
- /// is true, always compute the cost of scalarizing replicating operands.
- InstructionCost
- getScalarizationOverhead(Type *ResultTy, ArrayRef<const VPValue *> Operands,
- ElementCount VF,
- bool AlwaysIncludeReplicatingR = false);
+ /// type-based getScalarizationOverhead API. \p VIC provides context about
+ /// whether the scalarization is for a load/store operation. If \p
+ /// AlwaysIncludeReplicatingR is true, always compute the cost of scalarizing
+ /// replicating operands.
+ InstructionCost getScalarizationOverhead(
+ Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
+ TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None,
+ bool AlwaysIncludeReplicatingR = false);
};
/// This class can be used to assign names to VPValues. For VPValues without
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 64890ac9e9b92..a440a8f3f01ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3502,8 +3502,11 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
ResultTy = Ctx.Types.inferScalarType(this);
}
+ TTI::VectorInstrContext VIC =
+ IsLoad ? TTI::VectorInstrContext::Load : TTI::VectorInstrContext::Store;
return (ScalarCost * VF.getFixedValue()) +
- Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, true);
+ Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, VIC,
+ true);
}
case Instruction::SExt:
case Instruction::ZExt:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 4e7ed1f5a4ab7..e3c2a062a8b97 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -103,6 +103,7 @@ inline VPIRFlags getFlagsFromIndDesc(const InductionDescriptor &ID) {
"Expected int induction");
return VPIRFlags::WrapFlagsTy(false, false);
}
+
} // namespace vputils
//===----------------------------------------------------------------------===//
@@ -254,7 +255,6 @@ class VPBlockUtils {
/// Returns true if \p VPB is a loop latch, using isHeader().
static bool isLatch(const VPBlockBase *VPB, const VPDominatorTree &VPDT);
};
-
} // namespace llvm
#endif
>From 8fbbd294243c5da5d784f9682aa4c9488eaef9e5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 8 Dec 2025 17:09:25 +0000
Subject: [PATCH 2/2] [AArch64] Use VectorInstrContext in
getScalarizationOverhead.
Use VectorInstrContext to return more accurate scalarization overhead
costs when inserts/extracts can be folded into ld1/st1 and CPUs where
ld1/st1 are fast (same perf as regular loads).
Depends on https://github.com/llvm/llvm-project/pull/175982 (included in
PR).
---
.../AArch64/AArch64TargetTransformInfo.cpp | 9 +
.../replicating-load-store-costs-apple.ll | 272 +++++++++++++++---
2 files changed, 240 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index fade6f0bd17f7..3af47ba52fd0b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4210,6 +4210,15 @@ InstructionCost AArch64TTIImpl::getScalarizationOverhead(
TTI::VectorInstrContext VIC) const {
if (isa<ScalableVectorType>(Ty))
return InstructionCost::getInvalid();
+
+ // There's no scalarization overhead if ld1/st1 is cheap and the
+ // insert/extracts can be folded into the load/stores.
+ if (ST->hasFastLD1Single()) {
+ if ((VIC == TTI::VectorInstrContext::Store && Extract) ||
+ (VIC == TTI::VectorInstrContext::Load && Insert))
+ return 0;
+ }
+
if (Ty->getElementType()->isFloatingPointTy())
return BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
CostKind);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index b439353444409..21dc588253e46 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -368,53 +368,211 @@ declare void @init(ptr)
define void @scalar_store_cost_after_discarding_interleave_group(ptr %dst, i32 %x, ptr %src) {
; CHECK-LABEL: define void @scalar_store_cost_after_discarding_interleave_group(
; CHECK-SAME: ptr [[DST:%.*]], i32 [[X:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TEMP1:%.*]] = alloca [64 x i32], align 4
; CHECK-NEXT: call void @init(ptr [[TEMP1]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[TMP21:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = lshr <8 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT]], splat (i32 -171254)
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <8 x i32> [[TMP1]], splat (i32 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <8 x i32> [[TMP3]], splat (i32 1)
+; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], splat (i32 1)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc <8 x i32> [[TMP5]] to <8 x i16>
+; CHECK-NEXT: [[TMP7:%.*]] = sub <8 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = lshr <8 x i32> [[TMP7]], splat (i32 1)
+; CHECK-NEXT: [[TMP9:%.*]] = trunc <8 x i32> [[TMP8]] to <8 x i16>
+; CHECK-NEXT: [[TMP10:%.*]] = or <8 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
+; CHECK-NEXT: [[TMP11:%.*]] = add <8 x i32> [[TMP10]], splat (i32 1)
+; CHECK-NEXT: [[TMP12:%.*]] = lshr <8 x i32> [[TMP11]], splat (i32 1)
+; CHECK-NEXT: [[TMP13:%.*]] = trunc <8 x i32> [[TMP12]] to <8 x i16>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 24
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 32
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], 40
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 48
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[OFFSET_IDX]], 56
; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TEMP1]], align 4
-; CHECK-NEXT: [[SHR_0:%.*]] = lshr i32 [[X]], 1
-; CHECK-NEXT: [[MUL_0:%.*]] = mul i32 [[X]], -171254
-; CHECK-NEXT: [[SHR_1:%.*]] = lshr i32 [[MUL_0]], 1
-; CHECK-NEXT: [[ADD_0:%.*]] = add i32 [[SHR_0]], [[SHR_1]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[TMP22]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP23]]
; CHECK-NEXT: store i16 0, ptr [[TMP30]], align 2
-; CHECK-NEXT: [[GEP_0_1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[GEP_0_1]], i64 14
+; CHECK-NEXT: store i16 0, ptr [[TMP24]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP25]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP26]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP27]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP28]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP29]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP31]], align 2
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP30]], i64 14
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP24]], i64 14
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP25]], i64 14
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[TMP26]], i64 14
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP27]], i64 14
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[TMP28]], i64 14
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[TMP29]], i64 14
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[TMP31]], i64 14
; CHECK-NEXT: store i16 0, ptr [[TMP38]], align 2
-; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], 1
-; CHECK-NEXT: [[SHR_2:%.*]] = lshr i32 [[ADD_1]], 1
-; CHECK-NEXT: [[TMP54:%.*]] = trunc i32 [[SHR_2]] to i16
+; CHECK-NEXT: store i16 0, ptr [[TMP32]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP33]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP34]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP35]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP36]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP37]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP39]], align 2
; CHECK-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[TMP30]], i64 2
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr i8, ptr [[TMP24]], i64 2
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[TMP25]], i64 2
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[TMP26]], i64 2
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP27]], i64 2
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[TMP28]], i64 2
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr i8, ptr [[TMP29]], i64 2
+; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP31]], i64 2
+; CHECK-NEXT: [[TMP54:%.*]] = extractelement <8 x i16> [[TMP6]], i32 0
; CHECK-NEXT: store i16 [[TMP54]], ptr [[TMP46]], align 2
-; CHECK-NEXT: [[SUB_0:%.*]] = sub i32 0, [[MUL_0]]
-; CHECK-NEXT: [[SHR_3:%.*]] = lshr i32 [[SUB_0]], 1
-; CHECK-NEXT: [[TMP70:%.*]] = trunc i32 [[SHR_3]] to i16
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <8 x i16> [[TMP6]], i32 1
+; CHECK-NEXT: store i16 [[TMP48]], ptr [[TMP40]], align 2
+; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x i16> [[TMP6]], i32 2
+; CHECK-NEXT: store i16 [[TMP49]], ptr [[TMP41]], align 2
+; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x i16> [[TMP6]], i32 3
+; CHECK-NEXT: store i16 [[TMP50]], ptr [[TMP42]], align 2
+; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x i16> [[TMP6]], i32 4
+; CHECK-NEXT: store i16 [[TMP51]], ptr [[TMP43]], align 2
+; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x i16> [[TMP6]], i32 5
+; CHECK-NEXT: store i16 [[TMP52]], ptr [[TMP44]], align 2
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <8 x i16> [[TMP6]], i32 6
+; CHECK-NEXT: store i16 [[TMP53]], ptr [[TMP45]], align 2
+; CHECK-NEXT: [[TMP55:%.*]] = extractelement <8 x i16> [[TMP6]], i32 7
+; CHECK-NEXT: store i16 [[TMP55]], ptr [[TMP47]], align 2
; CHECK-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[TMP30]], i64 12
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[TMP24]], i64 12
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[TMP25]], i64 12
+; CHECK-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[TMP26]], i64 12
+; CHECK-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[TMP27]], i64 12
+; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[TMP28]], i64 12
+; CHECK-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[TMP29]], i64 12
+; CHECK-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[TMP31]], i64 12
+; CHECK-NEXT: [[TMP70:%.*]] = extractelement <8 x i16> [[TMP9]], i32 0
; CHECK-NEXT: store i16 [[TMP70]], ptr [[TMP62]], align 2
-; CHECK-NEXT: [[OR_0:%.*]] = or i32 [[X]], 1
-; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[OR_0]], 1
-; CHECK-NEXT: [[SHR_4:%.*]] = lshr i32 [[ADD_2]], 1
-; CHECK-NEXT: [[TMP86:%.*]] = trunc i32 [[SHR_4]] to i16
+; CHECK-NEXT: [[TMP64:%.*]] = extractelement <8 x i16> [[TMP9]], i32 1
+; CHECK-NEXT: store i16 [[TMP64]], ptr [[TMP56]], align 2
+; CHECK-NEXT: [[TMP65:%.*]] = extractelement <8 x i16> [[TMP9]], i32 2
+; CHECK-NEXT: store i16 [[TMP65]], ptr [[TMP57]], align 2
+; CHECK-NEXT: [[TMP66:%.*]] = extractelement <8 x i16> [[TMP9]], i32 3
+; CHECK-NEXT: store i16 [[TMP66]], ptr [[TMP58]], align 2
+; CHECK-NEXT: [[TMP67:%.*]] = extractelement <8 x i16> [[TMP9]], i32 4
+; CHECK-NEXT: store i16 [[TMP67]], ptr [[TMP59]], align 2
+; CHECK-NEXT: [[TMP68:%.*]] = extractelement <8 x i16> [[TMP9]], i32 5
+; CHECK-NEXT: store i16 [[TMP68]], ptr [[TMP60]], align 2
+; CHECK-NEXT: [[TMP69:%.*]] = extractelement <8 x i16> [[TMP9]], i32 6
+; CHECK-NEXT: store i16 [[TMP69]], ptr [[TMP61]], align 2
+; CHECK-NEXT: [[TMP71:%.*]] = extractelement <8 x i16> [[TMP9]], i32 7
+; CHECK-NEXT: store i16 [[TMP71]], ptr [[TMP63]], align 2
; CHECK-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[TMP30]], i64 4
+; CHECK-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[TMP24]], i64 4
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[TMP25]], i64 4
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[TMP26]], i64 4
+; CHECK-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[TMP27]], i64 4
+; CHECK-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[TMP28]], i64 4
+; CHECK-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[TMP29]], i64 4
+; CHECK-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[TMP31]], i64 4
+; CHECK-NEXT: [[TMP86:%.*]] = extractelement <8 x i16> [[TMP13]], i32 0
; CHECK-NEXT: store i16 [[TMP86]], ptr [[TMP78]], align 2
-; CHECK-NEXT: [[GEP_0_2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP94:%.*]] = getelementptr i8, ptr [[GEP_0_2]], i64 10
+; CHECK-NEXT: [[TMP80:%.*]] = extractelement <8 x i16> [[TMP13]], i32 1
+; CHECK-NEXT: store i16 [[TMP80]], ptr [[TMP72]], align 2
+; CHECK-NEXT: [[TMP81:%.*]] = extractelement <8 x i16> [[TMP13]], i32 2
+; CHECK-NEXT: store i16 [[TMP81]], ptr [[TMP73]], align 2
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <8 x i16> [[TMP13]], i32 3
+; CHECK-NEXT: store i16 [[TMP82]], ptr [[TMP74]], align 2
+; CHECK-NEXT: [[TMP83:%.*]] = extractelement <8 x i16> [[TMP13]], i32 4
+; CHECK-NEXT: store i16 [[TMP83]], ptr [[TMP75]], align 2
+; CHECK-NEXT: [[TMP84:%.*]] = extractelement <8 x i16> [[TMP13]], i32 5
+; CHECK-NEXT: store i16 [[TMP84]], ptr [[TMP76]], align 2
+; CHECK-NEXT: [[TMP85:%.*]] = extractelement <8 x i16> [[TMP13]], i32 6
+; CHECK-NEXT: store i16 [[TMP85]], ptr [[TMP77]], align 2
+; CHECK-NEXT: [[TMP87:%.*]] = extractelement <8 x i16> [[TMP13]], i32 7
+; CHECK-NEXT: store i16 [[TMP87]], ptr [[TMP79]], align 2
+; CHECK-NEXT: [[TMP94:%.*]] = getelementptr i8, ptr [[TMP30]], i64 10
+; CHECK-NEXT: [[TMP88:%.*]] = getelementptr i8, ptr [[TMP24]], i64 10
+; CHECK-NEXT: [[TMP89:%.*]] = getelementptr i8, ptr [[TMP25]], i64 10
+; CHECK-NEXT: [[TMP90:%.*]] = getelementptr i8, ptr [[TMP26]], i64 10
+; CHECK-NEXT: [[TMP91:%.*]] = getelementptr i8, ptr [[TMP27]], i64 10
+; CHECK-NEXT: [[TMP92:%.*]] = getelementptr i8, ptr [[TMP28]], i64 10
+; CHECK-NEXT: [[TMP93:%.*]] = getelementptr i8, ptr [[TMP29]], i64 10
+; CHECK-NEXT: [[TMP98:%.*]] = getelementptr i8, ptr [[TMP31]], i64 10
; CHECK-NEXT: store i16 0, ptr [[TMP94]], align 2
-; CHECK-NEXT: [[TRUNC_3:%.*]] = trunc i32 [[TMP22]] to i16
-; CHECK-NEXT: [[OR_1:%.*]] = or i16 [[TRUNC_3]], 1
-; CHECK-NEXT: [[TMP113:%.*]] = add i16 [[OR_1]], 1
+; CHECK-NEXT: store i16 0, ptr [[TMP88]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP89]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP90]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP91]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP92]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP93]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP98]], align 2
+; CHECK-NEXT: [[TMP95:%.*]] = trunc <8 x i32> [[BROADCAST_SPLAT2]] to <8 x i16>
+; CHECK-NEXT: [[TMP96:%.*]] = or <8 x i16> [[TMP95]], splat (i16 1)
+; CHECK-NEXT: [[TMP97:%.*]] = add <8 x i16> [[TMP96]], splat (i16 1)
+; CHECK-NEXT: [[TMP113:%.*]] = extractelement <8 x i16> [[TMP97]], i32 0
+; CHECK-NEXT: [[TMP99:%.*]] = extractelement <8 x i16> [[TMP97]], i32 1
+; CHECK-NEXT: [[TMP100:%.*]] = extractelement <8 x i16> [[TMP97]], i32 2
+; CHECK-NEXT: [[TMP101:%.*]] = extractelement <8 x i16> [[TMP97]], i32 3
+; CHECK-NEXT: [[TMP102:%.*]] = extractelement <8 x i16> [[TMP97]], i32 4
+; CHECK-NEXT: [[TMP103:%.*]] = extractelement <8 x i16> [[TMP97]], i32 5
+; CHECK-NEXT: [[TMP104:%.*]] = extractelement <8 x i16> [[TMP97]], i32 6
+; CHECK-NEXT: [[TMP106:%.*]] = extractelement <8 x i16> [[TMP97]], i32 7
; CHECK-NEXT: [[TMP105:%.*]] = getelementptr i8, ptr [[TMP30]], i64 8
+; CHECK-NEXT: [[TMP107:%.*]] = getelementptr i8, ptr [[TMP24]], i64 8
+; CHECK-NEXT: [[TMP108:%.*]] = getelementptr i8, ptr [[TMP25]], i64 8
+; CHECK-NEXT: [[TMP109:%.*]] = getelementptr i8, ptr [[TMP26]], i64 8
+; CHECK-NEXT: [[TMP110:%.*]] = getelementptr i8, ptr [[TMP27]], i64 8
+; CHECK-NEXT: [[TMP111:%.*]] = getelementptr i8, ptr [[TMP28]], i64 8
+; CHECK-NEXT: [[TMP112:%.*]] = getelementptr i8, ptr [[TMP29]], i64 8
+; CHECK-NEXT: [[TMP114:%.*]] = getelementptr i8, ptr [[TMP31]], i64 8
; CHECK-NEXT: store i16 [[TMP113]], ptr [[TMP105]], align 2
+; CHECK-NEXT: store i16 [[TMP99]], ptr [[TMP107]], align 2
+; CHECK-NEXT: store i16 [[TMP100]], ptr [[TMP108]], align 2
+; CHECK-NEXT: store i16 [[TMP101]], ptr [[TMP109]], align 2
+; CHECK-NEXT: store i16 [[TMP102]], ptr [[TMP110]], align 2
+; CHECK-NEXT: store i16 [[TMP103]], ptr [[TMP111]], align 2
+; CHECK-NEXT: store i16 [[TMP104]], ptr [[TMP112]], align 2
+; CHECK-NEXT: store i16 [[TMP106]], ptr [[TMP114]], align 2
; CHECK-NEXT: [[TMP121:%.*]] = getelementptr i8, ptr [[TMP30]], i64 6
+; CHECK-NEXT: [[TMP115:%.*]] = getelementptr i8, ptr [[TMP24]], i64 6
+; CHECK-NEXT: [[TMP116:%.*]] = getelementptr i8, ptr [[TMP25]], i64 6
+; CHECK-NEXT: [[TMP117:%.*]] = getelementptr i8, ptr [[TMP26]], i64 6
+; CHECK-NEXT: [[TMP118:%.*]] = getelementptr i8, ptr [[TMP27]], i64 6
+; CHECK-NEXT: [[TMP119:%.*]] = getelementptr i8, ptr [[TMP28]], i64 6
+; CHECK-NEXT: [[TMP120:%.*]] = getelementptr i8, ptr [[TMP29]], i64 6
+; CHECK-NEXT: [[TMP123:%.*]] = getelementptr i8, ptr [[TMP31]], i64 6
; CHECK-NEXT: store i16 0, ptr [[TMP121]], align 2
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[TMP21]], 8
-; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[TMP21]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: store i16 0, ptr [[TMP115]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP116]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP117]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP118]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP119]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP120]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP123]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP122:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: br i1 [[TMP122]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
;
entry:
%temp1 = alloca [64 x i32], align 4
@@ -503,21 +661,21 @@ define void @test_prefer_vector_addressing(ptr %start, ptr %ms, ptr noalias %src
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA12:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA12]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA14:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA14]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA14]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA14]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP15]]
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA17:![0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA17]]
-; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA17]]
+; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA19]]
+; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA19]]
+; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA19]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -618,7 +776,7 @@ define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.
; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP9]], [[TMP19]]
; CHECK-NEXT: [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP20]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -701,14 +859,16 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-LABEL: define i32 @test_or_reduction_with_stride_2(
; CHECK-SAME: i32 [[SCALE:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[SCALE]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
@@ -779,11 +939,41 @@ define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {
; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])
+; CHECK-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF25:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 48, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP68]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i32> poison, i32 [[SCALE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT1]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i32> [ [[TMP70]], %[[VEC_EPILOG_PH]] ], [ [[TMP81:%.*]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX3]], 2
+; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[OFFSET_IDX1]], 0
+; CHECK-NEXT: [[TMP72:%.*]] = add i64 [[OFFSET_IDX1]], 2
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP71]]
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP72]]
+; CHECK-NEXT: [[TMP75:%.*]] = load i8, ptr [[TMP73]], align 1
+; CHECK-NEXT: [[TMP76:%.*]] = load i8, ptr [[TMP74]], align 1
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <2 x i8> poison, i8 [[TMP75]], i32 0
+; CHECK-NEXT: [[TMP78:%.*]] = insertelement <2 x i8> [[TMP77]], i8 [[TMP76]], i32 1
+; CHECK-NEXT: [[TMP79:%.*]] = sext <2 x i8> [[TMP78]] to <2 x i32>
+; CHECK-NEXT: [[TMP80:%.*]] = mul <2 x i32> [[BROADCAST_SPLAT2]], [[TMP79]]
+; CHECK-NEXT: [[TMP81]] = or <2 x i32> [[TMP80]], [[VEC_PHI4]]
+; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX3]], 2
+; CHECK-NEXT: [[TMP82:%.*]] = icmp eq i64 [[INDEX_NEXT5]], 50
+; CHECK-NEXT: br i1 [[TMP82]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[SCALAR_PH]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP83:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP81]])
+; CHECK-NEXT: br i1 true, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
;
entry:
br label %loop
More information about the llvm-commits
mailing list