[llvm] [TTI] Remove masked/gather-scatter/strided/expand-compress costing from TTIImpl (PR #169885)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 2 23:28:14 PST 2025
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/169885
>From 4c6fd8acd00327d040c3b7a2fb4725672e0ce234 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 2 Dec 2025 22:13:04 -0800
Subject: [PATCH 1/3] [Hexagon][NFC] Drop no-op
getMaskedMemoryOpCost/getGatherScatterOpCost stubs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
These stubs (from 4bdf1aa416b02) don’t actually override anything.
Removing them eliminates the need for a local getMemIntrinsicCost()
forwarder in #169885.
---
.../Target/Hexagon/HexagonTargetTransformInfo.cpp | 12 ------------
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h | 6 ------
2 files changed, 18 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index c7d6ee306be84..59c6201e07081 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -223,12 +223,6 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
-InstructionCost
-HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
-}
-
InstructionCost
HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
VectorType *SrcTy, ArrayRef<int> Mask,
@@ -238,12 +232,6 @@ HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
return 1;
}
-InstructionCost
-HexagonTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
-}
-
InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 3135958ba4b50..edf88cf476f6d 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -120,16 +120,10 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
- InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
>From e58f28059a142f6972d56ce8b36afe5ed09c8be8 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 2 Dec 2025 22:48:44 -0800
Subject: [PATCH 2/3] [TTI] Use MemIntrinsicCostAttributes for getStridedOpCost
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 ++--
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 32 +++++++------------
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 18 ++++++++---
.../Target/RISCV/RISCVTargetTransformInfo.h | 8 ++---
4 files changed, 29 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 9b2a7f432a544..5f1d855621c93 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -864,10 +864,8 @@ class TargetTransformInfoImplBase {
}
virtual InstructionCost
- getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
+ getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
return InstructionCost::getInvalid();
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 314830652f0b6..fceff5f93b765 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1599,19 +1599,19 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
/*IsGatherScatter*/ true, CostKind);
}
- InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override {
+ InstructionCost
+ getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override {
// For a target without strided memory operations (or for an illegal
// operation type on one which does), assume we lower to a gather/scatter
// operation. (Which may in turn be scalarized.)
- unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_gather
- : Intrinsic::masked_scatter;
+ unsigned IID = MICA.getID() == Intrinsic::experimental_vp_strided_load
+ ? Intrinsic::masked_gather
+ : Intrinsic::masked_scatter;
return thisT()->getGatherScatterOpCost(
- MemIntrinsicCostAttributes(IID, DataTy, Ptr, VariableMask, Alignment,
- I),
+ MemIntrinsicCostAttributes(IID, MICA.getDataType(), MICA.getPointer(),
+ MICA.getVariableMask(), MICA.getAlignment(),
+ MICA.getInst()),
CostKind);
}
@@ -3062,21 +3062,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
unsigned Id = MICA.getID();
- Type *DataTy = MICA.getDataType();
- const Value *Ptr = MICA.getPointer();
- const Instruction *I = MICA.getInst();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
switch (Id) {
case Intrinsic::experimental_vp_strided_load:
- case Intrinsic::experimental_vp_strided_store: {
- unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
- ? Instruction::Load
- : Instruction::Store;
- return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
- }
+ case Intrinsic::experimental_vp_strided_store:
+ return thisT()->getStridedMemoryOpCost(MICA, CostKind);
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 1d431959eaea3..c740e263da024 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1212,14 +1212,22 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
-InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ unsigned Opcode = MICA.getID() == Intrinsic::experimental_vp_strided_load
+ ? Instruction::Load
+ : Instruction::Store;
+
+ Type *DataTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
!isLegalStridedLoadStore(DataTy, Alignment)) ||
(Opcode != Instruction::Load && Opcode != Instruction::Store))
- return BaseT::getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getStridedMemoryOpCost(MICA, CostKind);
if (CostKind == TTI::TCK_CodeSize)
return TTI::TCC_Basic;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index e32b1c553c57a..c1746e6d13166 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -202,11 +202,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
- InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost
+ getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override;
>From 3933c91e8e5129c75c093fecf5b676d0c800c39c Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 2 Dec 2025 23:15:49 -0800
Subject: [PATCH 3/3] [TTI] Remove
masked/gather-scatter/strided/expand-compress costing from TTIImpl
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 29 +----
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 108 +++++++-----------
.../AArch64/AArch64TargetTransformInfo.cpp | 18 ++-
.../AArch64/AArch64TargetTransformInfo.h | 12 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 18 ++-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 12 +-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 26 ++++-
.../Target/RISCV/RISCVTargetTransformInfo.h | 17 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 16 ++-
llvm/lib/Target/X86/X86TargetTransformInfo.h | 11 +-
10 files changed, 142 insertions(+), 125 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5f1d855621c93..fad08261e9981 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -845,30 +845,6 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return InstructionCost::getInvalid();
- }
-
virtual InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -928,6 +904,11 @@ class TargetTransformInfoImplBase {
virtual InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ unsigned IID = MICA.getID();
+ bool IsStrided = IID == Intrinsic::experimental_vp_strided_load ||
+ IID == Intrinsic::experimental_vp_strided_store;
+ if (IsStrided)
+ return InstructionCost::getInvalid();
return 1;
}
virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index fceff5f93b765..494199835a19c 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1557,64 +1557,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
- InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- Type *DataTy = MICA.getDataType();
- Align Alignment = MICA.getAlignment();
- unsigned Opcode = MICA.getID() == Intrinsic::masked_load
- ? Instruction::Load
- : Instruction::Store;
- // TODO: Pass on AddressSpace when we have test coverage.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
- CostKind);
- }
-
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
- MICA.getID() == Intrinsic::vp_gather)
- ? Instruction::Load
- : Instruction::Store;
- Type *DataTy = MICA.getDataType();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- true, CostKind);
- }
-
- InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
- ? Instruction::Load
- : Instruction::Store;
- Type *DataTy = MICA.getDataType();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
- // Treat expand load/compress store as gather/scatter operation.
- // TODO: implement more precise cost estimation for these intrinsics.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- /*IsGatherScatter*/ true, CostKind);
- }
-
- InstructionCost
- getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- // For a target without strided memory operations (or for an illegal
- // operation type on one which does), assume we lower to a gather/scatter
- // operation. (Which may in turn be scalarized.)
- unsigned IID = MICA.getID() == Intrinsic::experimental_vp_strided_load
- ? Intrinsic::masked_gather
- : Intrinsic::masked_scatter;
- return thisT()->getGatherScatterOpCost(
- MemIntrinsicCostAttributes(IID, MICA.getDataType(), MICA.getPointer(),
- MICA.getVariableMask(), MICA.getAlignment(),
- MICA.getInst()),
- CostKind);
- }
-
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -3062,22 +3004,56 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
unsigned Id = MICA.getID();
+ Type *DataTy = MICA.getDataType();
+ bool VariableMask = MICA.getVariableMask();
+ Align Alignment = MICA.getAlignment();
switch (Id) {
case Intrinsic::experimental_vp_strided_load:
- case Intrinsic::experimental_vp_strided_store:
- return thisT()->getStridedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::experimental_vp_strided_store: {
+ unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
+ ? Instruction::Load
+ : Instruction::Store;
+ // For a target without strided memory operations (or for an illegal
+ // operation type on one which does), assume we lower to a gather/scatter
+ // operation. (Which may in turn be scalarized.)
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
+ }
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
- case Intrinsic::vp_gather:
- return thisT()->getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::vp_gather: {
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
+ }
+ case Intrinsic::vp_load:
+ case Intrinsic::vp_store:
+ return InstructionCost::getInvalid();
case Intrinsic::masked_load:
- case Intrinsic::masked_store:
- return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_store: {
+ unsigned Opcode =
+ Id == Intrinsic::masked_load ? Instruction::Load : Instruction::Store;
+ // TODO: Pass on AddressSpace when we have test coverage.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
+ CostKind);
+ }
case Intrinsic::masked_compressstore:
- case Intrinsic::masked_expandload:
- return thisT()->getExpandCompressMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_expandload: {
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
+ ? Instruction::Load
+ : Instruction::Store;
+ // Treat expand load/compress store as gather/scatter operation.
+ // TODO: implement more precise cost estimation for these intrinsics.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask,
+ /*IsGatherScatter*/ true, CostKind);
+ }
case Intrinsic::vp_load_ff:
return InstructionCost::getInvalid();
default:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8b08b30388cc2..e8bbce202b407 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4746,13 +4746,27 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
return ST->hasSVE();
}
+InstructionCost
+AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
Type *Src = MICA.getDataType();
if (useNeonVector(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto LT = getTypeLegalizationCost(Src);
if (!LT.first.isValid())
return InstructionCost::getInvalid();
@@ -4808,7 +4822,7 @@ AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
const Instruction *I = MICA.getInst();
if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto *VT = cast<VectorType>(DataTy);
auto LT = getTypeLegalizationCost(DataTy);
if (!LT.first.isValid())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index c4f402b7c3b8e..7ca8482e152d1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -188,12 +188,14 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned Opcode2) const;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
Type *Src) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index bdaf9c5e7105c..88a7fb185bf16 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1636,6 +1636,20 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
CostKind, OpInfo, I);
}
+InstructionCost
+ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
@@ -1652,7 +1666,7 @@ ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
return ST->getMVEVectorCostFactor(CostKind);
}
if (!isa<FixedVectorType>(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Scalar cost, which is currently very high due to the efficiency of the
// generated code.
return cast<FixedVectorType>(Src)->getNumElements() * 8;
@@ -1711,7 +1725,7 @@ ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
using namespace PatternMatch;
if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
auto *VTy = cast<FixedVectorType>(DataTy);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index e5f9dd5fe26d9..a23256364dd9a 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -279,17 +279,19 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index c740e263da024..9a3a519ee2992 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1022,6 +1022,22 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
return getMemoryOpCost(Instruction::Load, DataTy, Alignment, AS, CostKind,
{TTI::OK_AnyValue, TTI::OP_None}, nullptr);
}
+ case Intrinsic::experimental_vp_strided_load:
+ case Intrinsic::experimental_vp_strided_store:
+ return getStridedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_compressstore:
+ case Intrinsic::masked_expandload:
+ return getExpandCompressMemoryOpCost(MICA, CostKind);
+ case Intrinsic::vp_scatter:
+ case Intrinsic::vp_gather:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::vp_load:
+ case Intrinsic::vp_store:
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
}
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
@@ -1037,7 +1053,7 @@ RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (!isLegalMaskedLoadStore(Src, Alignment) ||
CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
}
@@ -1150,13 +1166,13 @@ RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
Align Alignment = MICA.getAlignment();
const Instruction *I = MICA.getInst();
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if ((Opcode == Instruction::Load &&
!isLegalMaskedGather(DataTy, Align(Alignment))) ||
(Opcode == Instruction::Store &&
!isLegalMaskedScatter(DataTy, Align(Alignment))))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
@@ -1183,7 +1199,7 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
(Opcode == Instruction::Load &&
isLegalMaskedExpandLoad(DataTy, Alignment));
if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Example compressstore sequence:
// vsetivli zero, 8, e32, m2, ta, ma (ignored)
// vcompress.vm v10, v8, v0
@@ -1227,7 +1243,7 @@ RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
!isLegalStridedLoadStore(DataTy, Alignment)) ||
(Opcode != Instruction::Load && Opcode != Instruction::Store))
- return BaseT::getStridedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if (CostKind == TTI::TCK_CodeSize)
return TTI::TCC_Basic;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index c1746e6d13166..e6b75d7c458db 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -147,9 +147,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
- InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
@@ -194,17 +193,15 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ TTI::TargetCostKind CostKind) const;
- InstructionCost
- getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index aa75d2c60803e..9fb97918cb71a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5410,6 +5410,20 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
return Cost;
}
+InstructionCost
+X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
@@ -6275,7 +6289,7 @@ X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
(!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
Align(Alignment)))))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index d35911965d8b5..4f672793a0fcf 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -183,11 +183,12 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
const TTI::PointersChainInfo &Info, Type *AccessTy,
More information about the llvm-commits
mailing list