[llvm] [TTI] Use MemIntrinsicCostAttributes for getMaskedMemoryOpCost (PR #168029)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 16 18:47:35 PST 2025
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/168029
>From 6ed1673c844e3f91b613cad2722c16a3e4c4146e Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Thu, 13 Nov 2025 23:43:59 -0800
Subject: [PATCH 1/4] [TTI] Use MemIntrinsicCostAttributes for
getMaskedMemoryOpCost
- Split from #165532. This is a step toward a unified interface for
masked/gather-scatter/strided/expand-compress cost modeling.
- Replace the ad-hoc parameter list with a single attributes object.
API change:
// Before:
InstructionCost getMaskedMemoryOpCost(Opcode, Src, Alignment,
AddressSpace, CostKind);
// After:
InstructionCost getMaskedMemoryOpCost(MemIntrinsicCostAttributes,
CostKind);
Notes:
- NFCI intended: callers populate MemIntrinsicCostAttributes with
the same information as before.
- Follow-up: migrate gather/scatter, strided, and expand/compress
cost queries to the same attributes-based entry point.
---
.../llvm/Analysis/TargetTransformInfo.h | 49 ++++++++++++++++++-
.../llvm/Analysis/TargetTransformInfoImpl.h | 3 +-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 24 +++++----
llvm/lib/Analysis/TargetTransformInfo.cpp | 5 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 8 +--
.../AArch64/AArch64TargetTransformInfo.h | 3 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 14 +++---
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 3 +-
.../Hexagon/HexagonTargetTransformInfo.cpp | 6 +--
.../Hexagon/HexagonTargetTransformInfo.h | 3 +-
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 12 +++--
.../Target/RISCV/RISCVTargetTransformInfo.h | 3 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 19 ++++---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 6 ++-
.../Transforms/Vectorize/SLPVectorizer.cpp | 17 ++++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +--
17 files changed, 124 insertions(+), 62 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 0f17312b03827..a5b77a76fd92d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -123,6 +123,53 @@ struct HardwareLoopInfo {
LLVM_ABI bool canAnalyze(LoopInfo &LI);
};
+/// Information for memory intrinsic cost model.
+class MemIntrinsicCostAttributes {
+ /// Optional original context instruction, if one exists, e.g. the
+ /// load/store to transform to the intrinsic.
+ const Instruction *I = nullptr;
+
+ /// Address in memory.
+ const Value *Ptr = nullptr;
+
+ /// Vector type of the data to be loaded or stored.
+ Type *DataTy = nullptr;
+ Intrinsic::ID IID;
+
+ /// True when the memory access is predicated with a mask
+ /// that is not a compile-time constant.
+ bool VariableMask = true;
+ unsigned AddressSpace = 0;
+
+ /// Alignment of single element.
+ Align Alignment;
+
+public:
+ LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
+ const Value *Ptr, bool VariableMask,
+ Align Alignment,
+ const Instruction *I = nullptr)
+ : I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
+ Alignment(Alignment) {}
+ LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
+ Align Alignment, unsigned AddressSpace)
+ : DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
+ Alignment(Alignment) {}
+ LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
+ bool VariableMask, Align Alignment,
+ const Instruction *I = nullptr)
+ : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
+ Alignment(Alignment) {}
+
+ Intrinsic::ID getID() const { return IID; }
+ const Instruction *getInst() const { return I; }
+ const Value *getPointer() const { return Ptr; }
+ Type *getDataType() const { return DataTy; }
+ bool getVariableMask() const { return VariableMask; }
+ unsigned getAddressSpace() const { return AddressSpace; }
+ Align getAlignment() const { return Alignment; }
+};
+
class IntrinsicCostAttributes {
const IntrinsicInst *II = nullptr;
Type *RetTy = nullptr;
@@ -1556,7 +1603,7 @@ class TargetTransformInfo {
/// \return The cost of masked Load and Store instructions.
LLVM_ABI InstructionCost getMaskedMemoryOpCost(
- unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+ const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
/// \return The cost of Gather or Scatter operation
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index aacb88d2f9684..d8e35748f53e5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -842,8 +842,7 @@ class TargetTransformInfoImplBase {
}
virtual InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
return 1;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e8dbc964a943e..b74501040298a 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1547,9 +1547,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *DataTy, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
+ Type *DataTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_load
+ ? Instruction::Load
+ : Instruction::Store;
// TODO: Pass on AddressSpace when we have test coverage.
return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false,
CostKind);
@@ -1606,10 +1610,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
// Firstly, the cost of load/store operation.
InstructionCost Cost;
- if (UseMaskForCond || UseMaskForGaps)
- Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment,
- AddressSpace, CostKind);
- else
+ if (UseMaskForCond || UseMaskForGaps) {
+ unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ Cost = thisT()->getMaskedMemoryOpCost(
+ {IID, VecTy, Alignment, AddressSpace}, CostKind);
+ } else
Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace,
CostKind);
@@ -2408,14 +2414,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
case Intrinsic::masked_store: {
Type *Ty = Tys[0];
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
- return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0,
- CostKind);
+ return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
}
case Intrinsic::masked_load: {
Type *Ty = RetTy;
Align TyAlign = thisT()->DL.getABITypeAlign(Ty);
- return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0,
- CostKind);
+ return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind);
}
case Intrinsic::experimental_vp_strided_store: {
auto *Ty = cast<VectorType>(ICA.getArgTypes()[0]);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 0426ac7e62fab..45369f0ffe137 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1183,10 +1183,9 @@ InstructionCost TargetTransformInfo::getMemoryOpCost(
}
InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
- unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+ const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
- AddressSpace, CostKind);
+ InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(MICA, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 10f2c80edc1b3..9dbca001a3ff6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4720,12 +4720,12 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const {
}
InstructionCost
-AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
- Align Alignment, unsigned AddressSpace,
+AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ Type *Src = MICA.getDataType();
+
if (useNeonVector(Src))
- return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind);
+ return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
auto LT = getTypeLegalizationCost(Src);
if (!LT.first.isValid())
return InstructionCost::getInvalid();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index e3b0a1bec53ec..e04a5522e392c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -188,8 +188,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned Opcode2) const;
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
InstructionCost
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 9b250e6cac3ab..9b80c4f95ccff 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1631,20 +1631,22 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
}
InstructionCost
-ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ unsigned IID = MICA.getID();
+ Type *Src = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ unsigned AddressSpace = MICA.getAddressSpace();
if (ST->hasMVEIntegerOps()) {
- if (Opcode == Instruction::Load &&
+ if (IID == Intrinsic::masked_load &&
isLegalMaskedLoad(Src, Alignment, AddressSpace))
return ST->getMVEVectorCostFactor(CostKind);
- if (Opcode == Instruction::Store &&
+ if (IID == Intrinsic::masked_store &&
isLegalMaskedStore(Src, Alignment, AddressSpace))
return ST->getMVEVectorCostFactor(CostKind);
}
if (!isa<FixedVectorType>(Src))
- return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind);
+ return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
// Scalar cost, which is currently very high due to the efficiency of the
// generated code.
return cast<FixedVectorType>(Src)->getNumElements() * 8;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 0810c5532ed91..919a6fc9fd0b0 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -275,8 +275,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
InstructionCost getInterleavedMemoryOpCost(
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index e925e041eb64e..8f3f0cc8abb01 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -224,11 +224,9 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
}
InstructionCost
-HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
- Align Alignment, unsigned AddressSpace,
+HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
- return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind);
+ return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
}
InstructionCost
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index cec2bf9656ffc..e95b5a10b76a7 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -120,8 +120,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 7bc0b5b394828..85d66392862d9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1001,13 +1001,17 @@ InstructionCost RISCVTTIImpl::getScalarizationOverhead(
}
InstructionCost
-RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
+ : Instruction::Store;
+ Type *Src = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ unsigned AddressSpace = MICA.getAddressSpace();
+
if (!isLegalMaskedLoadStore(Src, Alignment) ||
CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
- CostKind);
+ return BaseT::getMaskedMemoryOpCost(MICA, CostKind);
return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 6886e8964e29e..39c1173e2986c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -144,8 +144,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool shouldConsiderVectorizationRegPressure() const override { return true; }
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
InstructionCost
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0b1430e373fc7..4b77bf925b2ba 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5411,9 +5411,14 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
}
InstructionCost
-X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
- unsigned AddressSpace,
+X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
+ unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
+ : Instruction::Store;
+ Type *SrcTy = MICA.getDataType();
+ Align Alignment = MICA.getAlignment();
+ unsigned AddressSpace = MICA.getAddressSpace();
+
bool IsLoad = (Instruction::Load == Opcode);
bool IsStore = (Instruction::Store == Opcode);
@@ -6647,10 +6652,12 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
LegalVT.getVectorNumElements());
InstructionCost MemOpCost;
bool UseMaskedMemOp = UseMaskForCond || UseMaskForGaps;
- if (UseMaskedMemOp)
- MemOpCost = getMaskedMemoryOpCost(Opcode, SingleMemOpTy, Alignment,
- AddressSpace, CostKind);
- else
+ if (UseMaskedMemOp) {
+ unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ MemOpCost = getMaskedMemoryOpCost(
+ {IID, SingleMemOpTy, Alignment, AddressSpace}, CostKind);
+ } else
MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, Alignment, AddressSpace,
CostKind);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index de5e1c297b1e4..df1393ce16ca1 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -183,8 +183,7 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost
- getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
- unsigned AddressSpace,
+ getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;
InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
const Value *Ptr, bool VariableMask,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e5c3f17860103..dd984c485c1ee 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5197,8 +5197,10 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
const Align Alignment = getLoadStoreAlignment(I);
InstructionCost Cost = 0;
if (Legal->isMaskRequired(I)) {
- Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
- CostKind);
+ unsigned IID = I->getOpcode() == Instruction::Load
+ ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ Cost += TTI.getMaskedMemoryOpCost({IID, VectorTy, Alignment, AS}, CostKind);
} else {
TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bf3f52c51b64c..f7a800652c551 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6848,9 +6848,10 @@ static bool isMaskedLoadCompress(
ScalarLoadsCost;
InstructionCost LoadCost = 0;
if (IsMasked) {
- LoadCost =
- TTI.getMaskedMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
- LI->getPointerAddressSpace(), CostKind);
+ LoadCost = TTI.getMaskedMemoryOpCost({Intrinsic::masked_load, LoadVecTy,
+ CommonAlignment,
+ LI->getPointerAddressSpace()},
+ CostKind);
} else {
LoadCost =
TTI.getMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment,
@@ -7249,8 +7250,9 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
break;
case LoadsState::CompressVectorize:
VecLdCost += TTI.getMaskedMemoryOpCost(
- Instruction::Load, SubVecTy, CommonAlignment,
- LI0->getPointerAddressSpace(), CostKind) +
+ {Intrinsic::masked_load, SubVecTy, CommonAlignment,
+ LI0->getPointerAddressSpace()},
+ CostKind) +
VectorGEPCost +
::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy,
{}, CostKind);
@@ -15041,8 +15043,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
CommonAlignment, LI0->getPointerAddressSpace(), CostKind);
} else if (IsMasked) {
VecLdCost = TTI->getMaskedMemoryOpCost(
- Instruction::Load, LoadVecTy, CommonAlignment,
- LI0->getPointerAddressSpace(), CostKind);
+ {Intrinsic::masked_load, LoadVecTy, CommonAlignment,
+ LI0->getPointerAddressSpace()},
+ CostKind);
// TODO: include this cost into CommonCost.
VecLdCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
LoadVecTy, CompressMask, CostKind);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1ee405a62aa68..aa84d68be729b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3566,8 +3566,10 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
InstructionCost Cost = 0;
if (IsMasked) {
+ unsigned IID = isa<VPWidenLoadRecipe>(this) ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
Cost +=
- Ctx.TTI.getMaskedMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind);
+ Ctx.TTI.getMaskedMemoryOpCost({IID, Ty, Alignment, AS}, Ctx.CostKind);
} else {
TTI::OperandValueInfo OpInfo = Ctx.getOperandInfo(
isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this) ? getOperand(0)
@@ -3686,7 +3688,7 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- Instruction::Load, Ty, Alignment, AS, Ctx.CostKind);
+ {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind);
if (!Reverse)
return Cost;
@@ -3795,7 +3797,7 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- Instruction::Store, Ty, Alignment, AS, Ctx.CostKind);
+ {Intrinsic::masked_store, Ty, Alignment, AS}, Ctx.CostKind);
if (!Reverse)
return Cost;
>From 5c53a465a32ae9d6edd8d16bd407d78428ea2999 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 14 Nov 2025 06:25:39 -0800
Subject: [PATCH 2/4] Address comments
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index a5b77a76fd92d..b2ed523c89aed 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -125,7 +125,7 @@ struct HardwareLoopInfo {
/// Information for memory intrinsic cost model.
class MemIntrinsicCostAttributes {
- /// Optional original context instruction, if one exists, e.g. the
+ /// Optional context instruction, if one exists, e.g. the
/// load/store to transform to the intrinsic.
const Instruction *I = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index aa84d68be729b..7ce60bb5bbaa4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3688,7 +3688,7 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind);
+ {Intrinsic::vp_load, Ty, Alignment, AS}, Ctx.CostKind);
if (!Reverse)
return Cost;
>From 41c8506e545199534d90bfd52407fc583d51f652 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sun, 16 Nov 2025 18:36:07 -0800
Subject: [PATCH 3/4] Keep subset of MICA
---
.../llvm/Analysis/TargetTransformInfo.h | 27 +++----------------
1 file changed, 3 insertions(+), 24 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index b2ed523c89aed..a65e4667ab76c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -125,47 +125,26 @@ struct HardwareLoopInfo {
/// Information for memory intrinsic cost model.
class MemIntrinsicCostAttributes {
- /// Optional context instruction, if one exists, e.g. the
- /// load/store to transform to the intrinsic.
- const Instruction *I = nullptr;
-
- /// Address in memory.
- const Value *Ptr = nullptr;
-
/// Vector type of the data to be loaded or stored.
Type *DataTy = nullptr;
+
+ /// ID of the memory intrinsic.
Intrinsic::ID IID;
- /// True when the memory access is predicated with a mask
- /// that is not a compile-time constant.
- bool VariableMask = true;
+ /// Address space of the pointer.
unsigned AddressSpace = 0;
/// Alignment of single element.
Align Alignment;
public:
- LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
- const Value *Ptr, bool VariableMask,
- Align Alignment,
- const Instruction *I = nullptr)
- : I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
- Alignment(Alignment) {}
LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
Align Alignment, unsigned AddressSpace)
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
Alignment(Alignment) {}
- LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
- bool VariableMask, Align Alignment,
- const Instruction *I = nullptr)
- : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
- Alignment(Alignment) {}
Intrinsic::ID getID() const { return IID; }
- const Instruction *getInst() const { return I; }
- const Value *getPointer() const { return Ptr; }
Type *getDataType() const { return DataTy; }
- bool getVariableMask() const { return VariableMask; }
unsigned getAddressSpace() const { return AddressSpace; }
Align getAlignment() const { return Alignment; }
};
>From d44b7e2a7a75c7913d282e765c8103f8624c001a Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sun, 16 Nov 2025 18:40:33 -0800
Subject: [PATCH 4/4] Add FIXME to VPlanRecipes
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7ce60bb5bbaa4..373aae57d7ebc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3687,8 +3687,10 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
+ // FIXME: getMaskedMemoryOpCost assumes masked_* intrinsics.
+ // After migrating to getMemIntrinsicInstrCost, switch this to vp_load.
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
- {Intrinsic::vp_load, Ty, Alignment, AS}, Ctx.CostKind);
+ {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind);
if (!Reverse)
return Cost;
@@ -3796,6 +3798,8 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
+ // FIXME: getMaskedMemoryOpCost assumes masked_* intrinsics.
+ // After migrating to getMemIntrinsicInstrCost, switch this to vp_load.
InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
{Intrinsic::masked_store, Ty, Alignment, AS}, Ctx.CostKind);
if (!Reverse)
More information about the llvm-commits
mailing list