[llvm] [SLP][NFC]Cache some results to improve compile time, NFC (PR #194599)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 04:32:11 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Try to avoid some recalculations, if possible to cache some previous
results
---
Full diff: https://github.com/llvm/llvm-project/pull/194599.diff
1 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+62-27)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9cf617beb40a4..3e40d49104315 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7693,13 +7693,23 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
auto *VecTy = getWidenedType(ScalarTy, Sz);
Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
+ // Cache masked gather legality - both the !IsSorted path below and the
+ // post-branch check use the same VecTy/CommonAlignment, and the underlying
+ // TTI calls are virtual.
+ std::optional<bool> MaskedGatherLegal;
+ auto IsMaskedGatherLegal = [&] {
+ if (!MaskedGatherLegal)
+ MaskedGatherLegal =
+ TTI->isLegalMaskedGather(VecTy, CommonAlignment) &&
+ !TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment);
+ return *MaskedGatherLegal;
+ };
if (!IsSorted) {
if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
SPtrInfo, /*isLoad=*/true))
return LoadsState::StridedVectorize;
- if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
- TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment))
+ if (!IsMaskedGatherLegal())
return LoadsState::Gather;
if (!all_of(PointerOps, [&](Value *P) {
@@ -7743,8 +7753,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
Diff, Ptr0, SPtrInfo))
return LoadsState::StridedVectorize;
}
- if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
- TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment))
+ if (!IsMaskedGatherLegal())
return LoadsState::Gather;
// Correctly identify compare the cost of loads + shuffles rather than
// strided/masked gather loads. Returns true if vectorized + shuffles
@@ -7764,12 +7773,14 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
APInt DemandedElts = APInt::getAllOnes(Sz);
Type *PtrScalarTy = PointerOps.front()->getType()->getScalarType();
VectorType *PtrVecTy = getWidenedType(PtrScalarTy, Sz);
+ // Cache the underlying object of PointerOps.front() - it is invariant
+ // across the per-V comparisons below and getUnderlyingObject walks
+ // GEP/cast chains.
+ const Value *FrontUO = getUnderlyingObject(PointerOps.front());
if (static_cast<unsigned>(count_if(
PointerOps, IsaPred<GetElementPtrInst>)) < PointerOps.size() - 1 ||
- any_of(PointerOps, [&](Value *V) {
- return getUnderlyingObject(V) !=
- getUnderlyingObject(PointerOps.front());
- }))
+ any_of(PointerOps,
+ [&](Value *V) { return getUnderlyingObject(V) != FrontUO; }))
VectorGEPCost += getScalarizationOverhead(TTI, PtrScalarTy, PtrVecTy,
DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
@@ -7873,8 +7884,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
count_if(PointerOps, IsaPred<GetElementPtrInst>)) <
PointerOps.size() - 1 ||
any_of(PointerOps, [&](Value *V) {
- return getUnderlyingObject(V) !=
- getUnderlyingObject(PointerOps.front());
+ return getUnderlyingObject(V) != FrontUO;
}))
VectorGEPCost += getScalarizationOverhead(
TTI, ScalarTy, SubVecTy, APInt::getAllOnes(SliceVF),
@@ -8722,6 +8732,12 @@ void BoUpSLP::reorderTopToBottom() {
// Maps a TreeEntry to the reorder indices of external users.
DenseMap<const TreeEntry *, SmallVector<OrdersType, 1>>
ExternalUserReorderMap;
+ // Compute IgnoreReorder once - it depends only on UserIgnoreList and
+ // VectorizableTree.front(), which do not change during this loop.
+ const bool IgnoreReorder =
+ !UserIgnoreList && VectorizableTree.front()->hasState() &&
+ (VectorizableTree.front()->getOpcode() == Instruction::InsertElement ||
+ VectorizableTree.front()->getOpcode() == Instruction::Store);
// Find all reorderable nodes with the given VF.
// Currently the are vectorized stores,loads,extracts + some gathering of
// extracts.
@@ -8755,10 +8771,6 @@ void BoUpSLP::reorderTopToBottom() {
// TODO: Check the reverse order too.
}
- bool IgnoreReorder =
- !UserIgnoreList && VectorizableTree.front()->hasState() &&
- (VectorizableTree.front()->getOpcode() == Instruction::InsertElement ||
- VectorizableTree.front()->getOpcode() == Instruction::Store);
if (std::optional<OrdersType> CurrentOrder =
getReorderingData(*TE, /*TopToBottom=*/true, IgnoreReorder)) {
// Do not include ordering for nodes used in the alt opcode vectorization,
@@ -12396,6 +12408,10 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
// Check if this is a duplicate of another entry.
LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
+ // Cache invariants to avoid recomputing for every V in VL (and every E).
+ const bool IsPHIWithLoop =
+ S.getOpcode() == Instruction::PHI &&
+ LI->getLoopFor(S.getMainOp()->getParent()) != nullptr;
for (TreeEntry *E : getTreeEntries(S.getMainOp())) {
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
@@ -12405,9 +12421,7 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V) ||
- (S.getOpcode() == Instruction::PHI && isa<PHINode>(V) &&
- LI->getLoopFor(S.getMainOp()->getParent()) &&
- isVectorized(V));
+ (IsPHIWithLoop && isa<PHINode>(V) && isVectorized(V));
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
@@ -14645,11 +14659,18 @@ void BoUpSLP::transformNodes() {
areKnownNonVectorizableLoads(Slice)))
continue;
if (VF == 2) {
+ // Cache the cost check lazily - both branches below may need it.
+ std::optional<bool> MainOpIsCheap;
+ auto IsMainOpCheap = [&] {
+ if (!MainOpIsCheap)
+ MainOpIsCheap =
+ TTI->getInstructionCost(S.getMainOp(), CostKind) <
+ TTI::TCC_Expensive;
+ return *MainOpIsCheap;
+ };
// Try to vectorize reduced values or if all users are vectorized.
// For expensive instructions extra extracts might be profitable.
- if ((!UserIgnoreList || E.Idx != 0) &&
- TTI->getInstructionCost(S.getMainOp(), CostKind) <
- TTI::TCC_Expensive &&
+ if ((!UserIgnoreList || E.Idx != 0) && IsMainOpCheap() &&
!all_of(Slice, [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
@@ -14679,8 +14700,7 @@ void BoUpSLP::transformNodes() {
continue;
}
} else if (S.getOpcode() == Instruction::ExtractElement ||
- (TTI->getInstructionCost(S.getMainOp(), CostKind) <
- TTI::TCC_Expensive &&
+ (IsMainOpCheap() &&
!CheckOperandsProfitability(
S.getMainOp(),
cast<Instruction>(*find_if(reverse(Slice),
@@ -17826,18 +17846,33 @@ InstructionCost BoUpSLP::getSpillCost() {
EntriesToOperands[TEPtr->UserTreeIndex.UserTE].push_back(TEPtr.get());
}
- auto NoCallIntrinsic = [this](const Instruction *I) {
- const auto *II = dyn_cast<IntrinsicInst>(I);
+ // Cache NoCallIntrinsicOrDoesNotReturn results - the same intrinsic call may
+ // be queried many times during the spill cost scan, and each computation
+ // involves two potentially expensive TTI virtual calls.
+ SmallDenseMap<const IntrinsicInst *, bool> NoCallIntrinsicCache;
+ auto NoCallIntrinsicOrDoesNotReturn = [this, &NoCallIntrinsicCache](
+ const Instruction *I) {
+ const auto *CB = dyn_cast<CallBase>(I);
+ if (!CB)
+ return false;
+ if (CB->doesNotReturn())
+ return true;
+ const auto *II = dyn_cast<IntrinsicInst>(CB);
if (!II)
return false;
if (II->isAssumeLikeIntrinsic())
return true;
+ auto [It, Inserted] = NoCallIntrinsicCache.try_emplace(II);
+ if (!Inserted)
+ return It->second;
IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II);
InstructionCost IntrCost =
TTI->getIntrinsicInstrCost(ICA, TTI::TCK_RecipThroughput);
InstructionCost CallCost = TTI->getCallInstrCost(
nullptr, II->getType(), ICA.getArgTypes(), TTI::TCK_RecipThroughput);
- return IntrCost < CallCost;
+ bool Res = IntrCost < CallCost;
+ It->second = Res;
+ return Res;
};
// Maps last instruction in the entry to the last instruction for the one of
@@ -17880,7 +17915,7 @@ InstructionCost BoUpSLP::getSpillCost() {
// Vectorized calls, represented as vector intrinsics, do not impact spill
// cost.
if (const auto *CB = dyn_cast<CallBase>(&*PrevInstIt);
- CB && !NoCallIntrinsic(CB) && !isVectorized(CB)) {
+ CB && !NoCallIntrinsicOrDoesNotReturn(CB) && !isVectorized(CB)) {
for (const Instruction *LastInst : LastInstsInRange)
CheckedInstructions.try_emplace(LastInst, &*PrevInstIt, 0);
return false;
@@ -17937,7 +17972,7 @@ InstructionCost BoUpSLP::getSpillCost() {
continue;
for (const Instruction &I : *BB) {
const auto *CB = dyn_cast<CallBase>(&I);
- if (!CB || NoCallIntrinsic(CB) || isVectorized(CB))
+ if (!CB || NoCallIntrinsicOrDoesNotReturn(CB) || isVectorized(CB))
continue;
LoopBodyHasNonVecCall.try_emplace(L, true);
return true;
``````````
</details>
https://github.com/llvm/llvm-project/pull/194599
More information about the llvm-commits
mailing list