[llvm] [TTI] Remove masked/gather-scatter/strided/expand-compress costing from TTIImpl (PR #169885)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 28 01:05:44 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Shih-Po Hung (arcbbb)
<details>
<summary>Changes</summary>
Following #<!-- -->165532, this patch moves scalarization‑cost computation into BaseT::getMemIntrinsicCost and lets backends override it via their getMemIntrinsicCost.
It also removes the masked/gather‑scatter/strided/expand‑compress costing interfaces from TTIImpl.
Targets may keep them locally if needed.
Stacked on #<!-- -->168650.
---
Patch is 32.19 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/169885.diff
12 Files Affected:
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (-28)
- (modified) llvm/include/llvm/CodeGen/BasicTTIImpl.h (+36-64)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+29-6)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+7-7)
- (modified) llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp (+26-6)
- (modified) llvm/lib/Target/ARM/ARMTargetTransformInfo.h (+7-7)
- (modified) llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp (+19-6)
- (modified) llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h (+6-7)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+56-16)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h (+10-13)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.cpp (+24-6)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.h (+7-7)
``````````diff
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 624302bc6d0a3..4af80e81fa9ff 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -845,34 +845,6 @@ class TargetTransformInfoImplBase {
return 1;
}
- virtual InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
- return 1;
- }
-
- virtual InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const {
- return 1;
- }
-
- virtual InstructionCost
- getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const {
- return InstructionCost::getInvalid();
- }
-
virtual InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index b1beb68feca46..a35c21a688f23 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1557,55 +1557,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}
- InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- Type *DataTy = MICA.getDataType();
- Align Alignment = MICA.getAlignment();
- unsigned Opcode = MICA.getID() == Intrinsic::masked_load
- ? Instruction::Load
- : Instruction::Store;
- // TODO: Pass on AddressSpace when we have test coverage.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
- CostKind);
- }
-
- InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override {
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- true, CostKind);
- }
-
- InstructionCost
- getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override {
- unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
- ? Instruction::Load
- : Instruction::Store;
- Type *DataTy = MICA.getDataType();
- bool VariableMask = MICA.getVariableMask();
- Align Alignment = MICA.getAlignment();
- // Treat expand load/compress store as gather/scatter operation.
- // TODO: implement more precise cost estimation for these intrinsics.
- return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
- /*IsGatherScatter*/ true, CostKind);
- }
-
- InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override {
- // For a target without strided memory operations (or for an illegal
- // operation type on one which does), assume we lower to a gather/scatter
- // operation. (Which may in turn be scalarized.)
- return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
- }
-
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -3056,26 +3007,47 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load
? Instruction::Load
: Instruction::Store;
- return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ // For a target without strided memory operations (or for an illegal
+ // operation type on one which does), assume we lower to a gather/scatter
+ // operation. (Which may in turn be scalarized.)
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
}
- case Intrinsic::masked_scatter:
- case Intrinsic::masked_gather:
case Intrinsic::vp_scatter:
- case Intrinsic::vp_gather: {
- unsigned Opcode =
- (Id == Intrinsic::masked_gather || Id == Intrinsic::vp_gather)
- ? Instruction::Load
- : Instruction::Store;
- return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ case Intrinsic::vp_gather:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather: {
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask, true, CostKind);
}
+
+ case Intrinsic::vp_load:
+ case Intrinsic::vp_store:
+ return InstructionCost::getInvalid();
case Intrinsic::masked_load:
- case Intrinsic::masked_store:
- return thisT()->getMaskedMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_store: {
+ unsigned Opcode =
+ Id == Intrinsic::masked_load ? Instruction::Load : Instruction::Store;
+ // TODO: Pass on AddressSpace when we have test coverage.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
+ CostKind);
+ }
case Intrinsic::masked_compressstore:
- case Intrinsic::masked_expandload:
- return thisT()->getExpandCompressMemoryOpCost(MICA, CostKind);
+ case Intrinsic::masked_expandload: {
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
+ ? Instruction::Load
+ : Instruction::Store;
+ // Treat expand load/compress store as gather/scatter operation.
+ // TODO: implement more precise cost estimation for these intrinsics.
+ return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment,
+ VariableMask,
+ /*IsGatherScatter*/ true, CostKind);
+ }
default:
llvm_unreachable("unexpected intrinsic");
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 52f0b75430d9d..f44dac68f3346 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4734,13 +4734,27 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
return ST->hasSVE();
}
+InstructionCost
+AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
Type *Src = MICA.getDataType();
if (useNeonVector(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto LT = getTypeLegalizationCost(Src);
if (!LT.first.isValid())
return InstructionCost::getInvalid();
@@ -4782,12 +4796,21 @@ static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
}
}
-InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
+ MICA.getID() == Intrinsic::vp_gather)
+ ? Instruction::Load
+ : Instruction::Store;
+
+ Type *DataTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy))
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
auto *VT = cast<VectorType>(DataTy);
auto LT = getTypeLegalizationCost(DataTy);
if (!LT.first.isValid())
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 52fc28a98449b..d22c79d8ef61a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -188,14 +188,14 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned Opcode2) const;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
- InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
Type *Src) const;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index fdb0ec40cb41f..a48589cd52bde 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1631,6 +1631,20 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
CostKind, OpInfo, I);
}
+InstructionCost
+ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
@@ -1647,7 +1661,7 @@ ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
return ST->getMVEVectorCostFactor(CostKind);
}
if (!isa<FixedVectorType>(Src))
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Scalar cost, which is currently very high due to the efficiency of the
// generated code.
return cast<FixedVectorType>(Src)->getNumElements() * 8;
@@ -1694,13 +1708,19 @@ InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
UseMaskForCond, UseMaskForGaps);
}
-InstructionCost ARMTTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
+InstructionCost
+ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+
+ Type *DataTy = MICA.getDataType();
+ const Value *Ptr = MICA.getPointer();
+ bool VariableMask = MICA.getVariableMask();
+ Align Alignment = MICA.getAlignment();
+ const Instruction *I = MICA.getInst();
+
using namespace PatternMatch;
if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
auto *VTy = cast<FixedVectorType>(DataTy);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 30f2151b41239..cefde6fc864eb 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -279,19 +279,19 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
+
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
- InstructionCost
- getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr,
- bool VariableMask, Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I = nullptr) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 3f84cbb6555ed..75262226d1e8f 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -223,10 +223,24 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
OpInfo, I);
}
+InstructionCost
+HexagonTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ switch (MICA.getID()) {
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_gather:
+ return getGatherScatterOpCost(MICA, CostKind);
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_store:
+ return getMaskedMemoryOpCost(MICA, CostKind);
+ }
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
+}
+
InstructionCost
HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
InstructionCost
@@ -238,11 +252,10 @@ HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
return 1;
}
-InstructionCost HexagonTTIImpl::getGatherScatterOpCost(
- unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
- Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
- return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
- Alignment, CostKind, I);
+InstructionCost
+HexagonTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const {
+ return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 67388984bb3e3..88bad14c63b86 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -120,18 +120,17 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
- TTI::TargetCostKind CostKind) const override;
+ getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const override;
+ InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
- InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- TTI::TargetCostKind CostKind,
- const Instruction *I) const override;
+ InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
+ TTI::TargetCostKind CostKind) const;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 4788a428d7e64..8aadb60f85a67 100644
--- a/llvm/lib/Target/RI...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/169885
More information about the llvm-commits
mailing list