[llvm] [TTI] Remove masked/gather-scatter/strided/expand-compress costing from TTIImpl (PR #169885)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 28 01:40:02 PST 2025
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/169885
>From 879f46099916134611d7b740fdd494607380cdd4 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 14 Nov 2025 06:19:43 -0800
Subject: [PATCH 1/8] [TTI] Use MemIntrinsicCostAttributes for
getGatherScatterOpCost
- Following from #168029. This is a step toward a unified interface for
masked/gather-scatter/strided/expand-compress cost modeling.
- Replace the ad-hoc parameter list with a single attributes object.
API change:
```
- InstructionCost getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, Inst);
+ InstructionCost getGatherScatterOpCost(MemIntrinsicCostAttributes,
+ CostKind);
```
Notes:
- NFCI intended: callers populate MemIntrinsicCostAttributes with
same information as before.
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 ++--
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 31 ++++++++++---------
.../AArch64/AArch64TargetTransformInfo.cpp | 19 +++++++++---
.../AArch64/AArch64TargetTransformInfo.h | 6 ++--
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 16 +++++++---
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 6 ++--
.../Hexagon/HexagonTargetTransformInfo.cpp | 9 +++---
.../Hexagon/HexagonTargetTransformInfo.h | 8 ++---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 19 +++++++-----
.../Target/RISCV/RISCVTargetTransformInfo.h | 8 ++---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 16 ++++++----
llvm/lib/Target/X86/X86TargetTransformInfo.h | 8 ++---
12 files changed, 83 insertions(+), 69 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 624302bc6d0a3..9b2a7f432a544 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -852,10 +852,8 @@ class TargetTransformInfoImplBase {
}
virtual InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
return 1;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index b1beb68feca46..dc0b729000881 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1571,10 +1571,15 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}
InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override {
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override {
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+ Type *DataTy = MICA.getDataType();
+ bool VariableMask = MICA.getVariableMask();
+ Align Alignment = MICA.getAlignment();
return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
true, CostKind);
}
@@ -1602,8 +1607,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// For a target without strided memory operations (or for an illegal
// operation type on one which does), assume we lower to a gather/scatter
// operation. (Which may in turn be scalarized.)
- return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_gather
+ : Intrinsic::masked_scatter;
+ return thisT()->getGatherScatterOpCost(
+ MemIntrinsicCostAttributes(IID, DataTy, Ptr, VariableMask, Alignment,
+ I),
+ CostKind);
}
InstructionCost getInterleavedMemoryOpCost(
@@ -3062,14 +3071,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
- case Intrinsic::vp_gather: {
- unsigned Opcode =
- (Id == Intrinsic::masked_gather || Id == Intrinsic::vp_gather)
- ? Instruction::Load
- : Instruction::Store;
- return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
- }
+ case Intrinsic::vp_gather:
+ return thisT()->getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 52f0b75430d9d..ad8c4f9974916 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4782,12 +4782,21 @@ static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
}
}
-InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+
+ Type *DataTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy))
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
auto *VT = cast<VectorType>(DataTy);
auto LT = getTypeLegalizationCost(DataTy);
if (!LT.first.isValid())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 52fc28a98449b..a15886b2a6afa 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -192,10 +192,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
TTI::TargetCostKind CostKind) const override;
InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override;
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
Type *Src) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index fdb0ec40cb41f..bb19c8415858f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1694,13 +1694,19 @@ InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
UseMaskForCond, UseMaskForGaps);
}
-InstructionCost ARMTTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ Type *DataTy = MICA.getDataType();
+ const Value *Ptr = MICA.getPointer();
+ bool VariableMask = MICA.getVariableMask();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
using namespace PatternMatch;
if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
auto *VTy = cast<FixedVectorType>(DataTy);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 30f2151b41239..e5f9dd5fe26d9 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -288,10 +288,8 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override;
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 3f84cbb6555ed..c7d6ee306be84 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -238,11 +238,10 @@ HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
return 1;
}
-InstructionCost HexagonTTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+InstructionCost
+HexagonTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
}
InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 67388984bb3e3..3135958ba4b50 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -127,11 +127,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 4788a428d7e64..64ed444b1a805 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1120,19 +1120,24 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
return MemCost + ShuffleCost;
}
-InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather;
+ unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
+ Type *DataTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
if ((Opcode == Instruction::Load &&
!isLegalMaskedGather(DataTy, Align(Alignment))) ||
(Opcode == Instruction::Store &&
!isLegalMaskedScatter(DataTy, Align(Alignment))))
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 5efa330b3ad71..b5822a139cc36 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -190,11 +190,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
InstructionCost
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index a9dc96b53d530..aa75d2c60803e 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6258,10 +6258,15 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
}
/// Calculate the cost of Gather / Scatter operation
-InstructionCost X86TTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
+InstructionCost
+X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather;
+ unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
+ Type *SrcVTy = MICA.getDataType();
+ const Value *Ptr = MICA.getPointer();
+ Align Alignment = MICA.getAlignment();
if ((Opcode == Instruction::Load &&
(!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
@@ -6270,8 +6275,7 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost(
(!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
Align(Alignment)))))
- return BaseT::getGatherScatterOpCost(Opcode, SrcVTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getGatherScatterOpCost(MICA, CostKind);
assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index d6dea9427990b..d35911965d8b5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -185,11 +185,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
InstructionCost
getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
- InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost
+ getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
InstructionCost
getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
const TTI::PointersChainInfo &Info, Type *AccessTy,
>From e1388aa081817f5f9166b1a279ddfb76a74bf10c Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Thu, 27 Nov 2025 22:17:49 -0800
Subject: [PATCH 2/8] [TTI] Remove getMaskedMemoryOpCost from
TargetTransformInfoImpl
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 -----
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 22 ++++++-------------
.../AArch64/AArch64TargetTransformInfo.cpp | 13 ++++++++++-
.../AArch64/AArch64TargetTransformInfo.h | 7 ++++--
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 13 ++++++++++-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 7 ++++--
.../Hexagon/HexagonTargetTransformInfo.cpp | 13 ++++++++++-
.../Hexagon/HexagonTargetTransformInfo.h | 6 +++--
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 13 ++++++++++-
.../Target/RISCV/RISCVTargetTransformInfo.h | 7 ++++--
.../lib/Target/X86/X86TargetTransformInfo.cpp | 11 ++++++++++
llvm/lib/Target/X86/X86TargetTransformInfo.h | 7 ++++--
12 files changed, 90 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 9b2a7f432a544..e67eb89363f1f 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -845,12 +845,6 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
virtual InstructionCost
getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index dc0b729000881..f2736d695108c 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1557,19 +1557,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
- InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- Type *DataTy = MICA.getDataType();
- Align Alignment = MICA.getAlignment();
- unsigned Opcode = MICA.getID() == Intrinsic::masked_load
- ? Instruction::Load
- : Instruction::Store;
- // TODO: Pass on AddressSpace when we have test coverage.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
- CostKind);
- }
-
InstructionCost
getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
@@ -3074,8 +3061,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::vp_gather:
return thisT()->getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
- case Intrinsic::masked_store:
- return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_store: {
+ unsigned Opcode =
+ Id == Intrinsic::masked_load ? Instruction::Load : Instruction::Store;
+ // TODO: Pass on AddressSpace when we have test coverage.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
+ CostKind);
+ }
case Intrinsic::masked_compressstore:
case Intrinsic::masked_expandload:
return thisT()->getExpandCompressMemoryOpCost(MICA, CostKind);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ad8c4f9974916..8eebd33c0ba49 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4734,13 +4734,24 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
return ST->hasSVE();
}
+InstructionCost
+AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
Type *Src = MICA.getDataType();
if (useNeonVector(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto LT = getTypeLegalizationCost(Src);
if (!LT.first.isValid())
return InstructionCost::getInvalid();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index a15886b2a6afa..b1a62bb15591f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -188,8 +188,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned Opcode2) const;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index bb19c8415858f..40611921dc124 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1631,6 +1631,17 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
CostKind, OpInfo, I);
}
+InstructionCost
+ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
@@ -1647,7 +1658,7 @@ ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
return ST->getMVEVectorCostFactor(CostKind);
}
if (!isa<FixedVectorType>(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Scalar cost, which is currently very high due to the efficiency of the
// generated code.
return cast<FixedVectorType>(Src)->getNumElements() * 8;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index e5f9dd5fe26d9..6affd951f9008 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -279,8 +279,11 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index c7d6ee306be84..753f47b4993fc 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -223,10 +223,21 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
+InstructionCost
+HexagonTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
InstructionCost
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 3135958ba4b50..e2a483d7add94 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -120,8 +120,10 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 64ed444b1a805..29205a2ba54cf 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1007,6 +1007,17 @@ InstructionCost RISCVTTIImpl::getScalarizationOverhead(
return Cost;
}
+InstructionCost
+RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
@@ -1018,7 +1029,7 @@ RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
if (!isLegalMaskedLoadStore(Src, Alignment) ||
CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index b5822a139cc36..dbf537bba7acc 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -144,8 +144,11 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool shouldConsiderVectorizationRegPressure() const override { return true; }
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index aa75d2c60803e..5ee445713ba73 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5410,6 +5410,17 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
return Cost;
}
+InstructionCost
+X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index d35911965d8b5..59ce33a0841a4 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -182,9 +182,12 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::TargetCostKind CostKind,
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
+
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
>From 52658f16df6a87c52bdef3a5d885a7c093de60a9 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Thu, 27 Nov 2025 22:55:58 -0800
Subject: [PATCH 3/8] Remove getStridedMemoryOpCost and
getExpandCompressMemoryOpCost
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 14 ------
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 48 ++++++-------------
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 28 ++++++++---
.../Target/RISCV/RISCVTargetTransformInfo.h | 9 ++--
4 files changed, 39 insertions(+), 60 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index e67eb89363f1f..521fa6129da3c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -851,20 +851,6 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
- return InstructionCost::getInvalid();
- }
-
virtual InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index f2736d695108c..dbf0a312dba02 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1571,37 +1571,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
true, CostKind);
}
- InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
- ? Instruction::Load
- : Instruction::Store;
- Type *DataTy = MICA.getDataType();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
- // Treat expand load/compress store as gather/scatter operation.
- // TODO: implement more precise cost estimation for these intrinsics.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- /*IsGatherScatter*/ true, CostKind);
- }
-
- InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override {
- // For a target without strided memory operations (or for an illegal
- // operation type on one which does), assume we lower to a gather/scatter
- // operation. (Which may in turn be scalarized.)
- unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_gather
- : Intrinsic::masked_scatter;
- return thisT()->getGatherScatterOpCost(
- MemIntrinsicCostAttributes(IID, DataTy, Ptr, VariableMask, Alignment,
- I),
- CostKind);
- }
-
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -3052,7 +3021,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
? Instruction::Load
: Instruction::Store;
- return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
+ // For a target without strided memory operations (or for an illegal
+ // operation type on one which does), assume we lower to a gather/scatter
+ // operation. (Which may in turn be scalarized.)
+ return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
Alignment, CostKind, I);
}
case Intrinsic::masked_scatter:
@@ -3069,8 +3041,16 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
CostKind);
}
case Intrinsic::masked_compressstore:
- case Intrinsic::masked_expandload:
- return thisT()->getExpandCompressMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_expandload: {
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
+ ? Instruction::Load
+ : Instruction::Store;
+ // Treat expand load/compress store as gather/scatter operation.
+ // TODO: implement more precise cost estimation for these intrinsics.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask,
+ /*IsGatherScatter*/ true, CostKind);
+ }
default:
llvm_unreachable("unexpected intrinsic");
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 29205a2ba54cf..24fcea19887d7 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1011,6 +1011,12 @@ InstructionCost
RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
switch (MICA.getID()) {
+ case Intrinsic::experimental_vp_strided_load:
+ case Intrinsic::experimental_vp_strided_store:
+ return getStridedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_compressstore:
+ case Intrinsic::masked_expandload:
+ return getExpandCompressMemoryOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -1175,7 +1181,7 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
(Opcode == Instruction::Load &&
isLegalMaskedExpandLoad(DataTy, Alignment));
if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Example compressstore sequence:
// vsetivli zero, 8, e32, m2, ta, ma (ignored)
// vcompress.vm v10, v8, v0
@@ -1204,14 +1210,24 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
-InstructionCost RISCVTTIImpl::getStridedMemoryOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ unsigned Opcode = MICA.getID() == Intrinsic::experimental_vp_strided_load
+ ? Instruction::Load
+ : Instruction::Store;
+
+ Type *DataTy = MICA.getDataType();
+ const Value *Ptr = MICA.getPointer();
+ bool VariableMask = MICA.getVariableMask();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
if (((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
!isLegalStridedLoadStore(DataTy, Alignment)) ||
(Opcode != Instruction::Load && Opcode != Instruction::Store))
- return BaseT::getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if (CostKind == TTI::TCK_CodeSize)
return TTI::TCC_Basic;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index dbf537bba7acc..4ee2c36ecf343 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -199,13 +199,10 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
InstructionCost
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ TTI::TargetCostKind CostKind) const;
- InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const override;
>From 125a74e9b6b1a7b6972e0f7bebc65c39c3dfcb04 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Thu, 27 Nov 2025 23:25:53 -0800
Subject: [PATCH 4/8] Remove getGatherScatterOpCost from
TargetTransformInfoImpl
---
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 ----
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 32 ++++++++-----------
.../AArch64/AArch64TargetTransformInfo.cpp | 5 ++-
.../AArch64/AArch64TargetTransformInfo.h | 5 ++-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 5 ++-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 5 ++-
.../Hexagon/HexagonTargetTransformInfo.cpp | 5 ++-
.../Hexagon/HexagonTargetTransformInfo.h | 5 ++-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 9 ++++--
.../Target/RISCV/RISCVTargetTransformInfo.h | 5 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 5 ++-
llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 ++-
12 files changed, 46 insertions(+), 46 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 521fa6129da3c..4af80e81fa9ff 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -845,12 +845,6 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
virtual InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index dbf0a312dba02..ffdaef49b1c78 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1557,20 +1557,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
- MICA.getID() == Intrinsic::vp_gather)
- ? Instruction::Load
- : Instruction::Store;
- Type *DataTy = MICA.getDataType();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- true, CostKind);
- }
-
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -3024,14 +3010,22 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// For a target without strided memory operations (or for an illegal
// operation type on one which does), assume we lower to a gather/scatter
// operation. (Which may in turn be scalarized.)
- return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
}
- case Intrinsic::masked_scatter:
- case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
case Intrinsic::vp_gather:
- return thisT()->getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather: {
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
+ }
+
case Intrinsic::masked_load:
case Intrinsic::masked_store: {
unsigned Opcode =
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8eebd33c0ba49..f44dac68f3346 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4738,6 +4738,9 @@ InstructionCost
AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -4807,7 +4810,7 @@ AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
const Instruction *I = MICA.getInst();
if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto *VT = cast<VectorType>(DataTy);
auto LT = getTypeLegalizationCost(DataTy);
if (!LT.first.isValid())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index b1a62bb15591f..d22c79d8ef61a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -194,9 +194,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
Type *Src) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 40611921dc124..a48589cd52bde 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1635,6 +1635,9 @@ InstructionCost
ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -1717,7 +1720,7 @@ ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
using namespace PatternMatch;
if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
auto *VTy = cast<FixedVectorType>(DataTy);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 6affd951f9008..cefde6fc864eb 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -290,9 +290,8 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 753f47b4993fc..75262226d1e8f 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -227,6 +227,9 @@ InstructionCost
HexagonTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -252,7 +255,7 @@ HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
InstructionCost
HexagonTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index e2a483d7add94..88bad14c63b86 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -129,9 +129,8 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 24fcea19887d7..3e0207b52eb2f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1017,6 +1017,11 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
case Intrinsic::masked_compressstore:
case Intrinsic::masked_expandload:
return getExpandCompressMemoryOpCost(MICA, CostKind);
+ case Intrinsic::vp_scatter:
+ case Intrinsic::vp_gather:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -1148,13 +1153,13 @@ RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
Align Alignment = MICA.getAlignment();
const Instruction *I = MICA.getInst();
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if ((Opcode == Instruction::Load &&
!isLegalMaskedGather(DataTy, Align(Alignment))) ||
(Opcode == Instruction::Store &&
!isLegalMaskedScatter(DataTy, Align(Alignment))))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 4ee2c36ecf343..e6b75d7c458db 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -193,9 +193,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 5ee445713ba73..9fb97918cb71a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5414,6 +5414,9 @@ InstructionCost
X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -6286,7 +6289,7 @@ X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
(!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
Align(Alignment)))))
- return BaseT::getGatherScatterOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 59ce33a0841a4..9b68909df9363 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -188,9 +188,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::TargetCostKind CostKind) const override;
InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const;
- InstructionCost
- getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
const TTI::PointersChainInfo &Info, Type *AccessTy,
>From 8ca414de3020ee0e62d7f416948b25eabc05e8af Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 28 Nov 2025 01:01:06 -0800
Subject: [PATCH 5/8] Handle vp_load/store
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 3 +++
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 7 +++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index ffdaef49b1c78..a35c21a688f23 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -3026,6 +3026,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
VariableMask, true, CostKind);
}
+ case Intrinsic::vp_load:
+ case Intrinsic::vp_store:
+ return InstructionCost::getInvalid();
case Intrinsic::masked_load:
case Intrinsic::masked_store: {
unsigned Opcode =
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 3e0207b52eb2f..8aadb60f85a67 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1022,6 +1022,8 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::vp_load:
+ case Intrinsic::vp_store:
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
@@ -1032,8 +1034,9 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
InstructionCost
RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
- : Instruction::Store;
+ unsigned IID = MICA.getID();
+ bool IsLoad = IID == Intrinsic::masked_load || IID == Intrinsic::vp_load;
+ unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
Type *Src = MICA.getDataType();
Align Alignment = MICA.getAlignment();
unsigned AddressSpace = MICA.getAddressSpace();
>From bc29d54157ec51f7c5273a0b08410784aa14fc49 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 28 Nov 2025 01:11:35 -0800
Subject: [PATCH 6/8] Fix basic cost for strided op
---
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 4af80e81fa9ff..fad08261e9981 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -904,6 +904,11 @@ class TargetTransformInfoImplBase {
virtual InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ unsigned IID = MICA.getID();
+ bool IsStrided = IID == Intrinsic::experimental_vp_strided_load ||
+ IID == Intrinsic::experimental_vp_strided_store;
+ if (IsStrided)
+ return InstructionCost::getInvalid();
return 1;
}
virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy,
>From a64aa81540e9fdeec8bcbc5ae2dfec4538c27c30 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 28 Nov 2025 01:25:34 -0800
Subject: [PATCH 7/8] Remove unused variable
---
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index a35c21a688f23..489fd7289a7b0 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2996,8 +2996,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
TTI::TargetCostKind CostKind) const override {
unsigned Id = MICA.getID();
Type *DataTy = MICA.getDataType();
- const Value *Ptr = MICA.getPointer();
- const Instruction *I = MICA.getInst();
bool VariableMask = MICA.getVariableMask();
Align Alignment = MICA.getAlignment();
>From 7718a87e91b7270872365ee355573fd1ee9ad2a0 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 28 Nov 2025 01:39:38 -0800
Subject: [PATCH 8/8] Fix build error for ARM
---
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index cefde6fc864eb..a23256364dd9a 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -280,7 +280,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
InstructionCost
getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const;
+ TTI::TargetCostKind CostKind) const override;
InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const;
More information about the llvm-commits
mailing list