[llvm] [SLP]Scale spill cost by the trip count of the loop with the call (PR #218123)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 04:46:48 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/218123
Hoisted loop-invariant vector values live over a call were charged the
keep-live cost only once, though the spill/reload executes every
iteration. Scale by the trip count of the loop containing the call.
Fixes https://github.com/llvm/llvm-project/issues/214555#issuecomment-5376445123
>From d4ecf0bb2d475d66914ee59f19ba45798f670030 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 22 Aug 2026 04:46:34 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 58 ++++++++++---------
.../AArch64/spillcost-loop-backedge.ll | 33 +++++------
.../RISCV/spillcost-loop-invariant-entry.ll | 22 +++----
3 files changed, 58 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d90c8f5fd336e..d6c3ae503caae 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18748,7 +18748,13 @@ InstructionCost BoUpSLP::getSpillCost() {
LastInst, Completed ? First : &*PrevInstIt, NoCallsInRange ? 1 : 0);
return NoCallsInRange;
};
- auto AddCosts = [&](const TreeEntry *Op) {
+ // The spill/reload is executed once per execution of the call, so the
+ // cost is scaled by the trip count of the loop containing the call, even
+ // for hoisted loop-invariant values defined outside of it.
+ auto GetSpillScale = [&](const BasicBlock *BB) {
+ return getLoopNestScale(LI->getLoopFor(BB));
+ };
+ auto AddCosts = [&](const TreeEntry *Op, uint64_t Scale) {
if (ScalarOrPseudoEntries.contains(Op))
return;
Type *ScalarTy = Op->Scalars.front()->getType();
@@ -18756,7 +18762,6 @@ InstructionCost BoUpSLP::getSpillCost() {
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, Op->getVectorFactor());
- uint64_t Scale = getScaleToLoopIterations(*Op);
InstructionCost KeepLiveCost = TTI->getCostOfKeepingLiveOverCall(VecTy);
KeepLiveCost *= Scale;
Cost += KeepLiveCost;
@@ -18766,10 +18771,11 @@ InstructionCost BoUpSLP::getSpillCost() {
Scale;
}
};
- // Memoize the relationship between blocks, i.e. if there is (at least one)
- // non-vectorized call between the blocks. This allows to skip the analysis of
- // the same block paths multiple times.
- SmallDenseMap<std::pair<const BasicBlock *, const BasicBlock *>, bool>
+ // Memoize the relationship between blocks, i.e. the spill scale if every
+ // path between the blocks crosses a non-vectorized call, 0 if there is (at
+ // least one) call-free path. This allows to skip the analysis of the same
+ // block paths multiple times.
+ SmallDenseMap<std::pair<const BasicBlock *, const BasicBlock *>, uint64_t>
ParentOpParentToPreds;
// Memoize whether a basic block contains a non-terminator no-return call.
// Such blocks are dead-end paths in normal control flow (execution does not
@@ -18823,7 +18829,7 @@ InstructionCost BoUpSLP::getSpillCost() {
if (auto It = ParentOpParentToPreds.find(Key);
It != ParentOpParentToPreds.end())
return It->second;
- bool Res = false;
+ uint64_t Res = 0;
scope_exit Cleanup([&]() { ParentOpParentToPreds.try_emplace(Key, Res); });
// If Op is loop-invariant, a call anywhere in the loop body forces a spill,
// even when a call-free forward path from Root back to OpParent exists on
@@ -18835,8 +18841,10 @@ InstructionCost BoUpSLP::getSpillCost() {
Outermost = L;
L = L->getParentLoop();
}
- if (Outermost && LoopBodyHasCall(Outermost))
+ if (Outermost && LoopBodyHasCall(Outermost)) {
+ Res = getLoopNestScale(Outermost);
return Res;
+ }
SmallVector<BasicBlock *> Worklist;
if (Pred)
Worklist.push_back(Pred);
@@ -18848,7 +18856,7 @@ InstructionCost BoUpSLP::getSpillCost() {
// blocks that were visited during the BFS are not necessarily
// call-free-reachable to OpParent themselves - we may have reached
// OpParent through a *sibling* path that bypassed them.
- // We return `true` (no spill cost) if at least one backward path from
+ // We return 0 (no spill cost) if at least one backward path from
// some predecessor of Root back to OpParent is call-free. Only when
// *every* such path goes through a non-vec call do we charge the spill
// cost: only then is it actually necessary to keep the vectorized value
@@ -18861,15 +18869,14 @@ InstructionCost BoUpSLP::getSpillCost() {
//
// If we ever pop OpParent from the worklist, we have reached it through
// a chain of call-free, non-dominated blocks: a call-free path exists
- // and we return true. If the worklist is exhausted without reaching
+ // and we return 0. If the worklist is exhausted without reaching
// OpParent, every admissible path is blocked by a call and we return
- // false so the caller charges the spill cost.
+ // the scale of the loop containing Root, so the caller charges the spill
+ // cost.
while (!Worklist.empty()) {
BasicBlock *BB = Worklist.pop_back_val();
- if (BB == OpParent) {
- Res = true;
+ if (BB == OpParent)
return Res;
- }
if (!Visited.insert(BB).second)
continue;
// Blocks strictly dominated by Root are reached only *after* Root in
@@ -18886,11 +18893,9 @@ InstructionCost BoUpSLP::getSpillCost() {
auto Pair = std::make_pair(BB, OpParent);
if (auto It = ParentOpParentToPreds.find(Pair);
It != ParentOpParentToPreds.end()) {
- if (It->second) {
- // BB is known to reach OpParent via a call-free path.
- Res = true;
+ // BB is known to reach OpParent via a call-free path.
+ if (It->second == 0)
return Res;
- }
// BB is known to be blocked from OpParent by calls; keep checking
// other paths.
continue;
@@ -18909,6 +18914,7 @@ InstructionCost BoUpSLP::getSpillCost() {
}
// Worklist drained without ever reaching OpParent: every path between
// Root and OpParent is blocked by a non-vec call.
+ Res = GetSpillScale(Root);
return Res;
};
SmallVector<const TreeEntry *> LiveEntries(1, Root);
@@ -18962,7 +18968,7 @@ InstructionCost BoUpSLP::getSpillCost() {
all_of(Op->Scalars, [&](Value *V) {
return !isa<Instruction>(V) || L->isLoopInvariant(V);
}))
- AddCosts(Op);
+ AddCosts(Op, GetSpillScale(Parent));
continue;
}
Budget = 0;
@@ -18995,11 +19001,11 @@ InstructionCost BoUpSLP::getSpillCost() {
if (OpParent == Parent) {
if (Entry->getOpcode() == Instruction::PHI) {
if (!CheckForNonVecCallsInSameBlock(LastInst, OpLastInst))
- AddCosts(Op);
+ AddCosts(Op, GetSpillScale(Parent));
continue;
}
if (!CheckForNonVecCallsInSameBlock(OpLastInst, LastInst))
- AddCosts(Op);
+ AddCosts(Op, GetSpillScale(Parent));
continue;
}
// Check for call instruction in between blocks.
@@ -19007,20 +19013,18 @@ InstructionCost BoUpSLP::getSpillCost() {
if (Entry->getOpcode() != Instruction::PHI &&
!CheckForNonVecCallsInSameBlock(
&*Parent->getFirstNonPHIOrDbgOrAlloca(), LastInst)) {
- AddCosts(Op);
+ AddCosts(Op, GetSpillScale(Parent));
continue;
}
// 2. Check op's block from the end.
if (!CheckForNonVecCallsInSameBlock(OpLastInst,
OpParent->getTerminator())) {
- AddCosts(Op);
+ AddCosts(Op, GetSpillScale(OpParent));
continue;
}
// 3. Check the predecessors of entry's block till op's block.
- if (!CheckPredecessors(Parent, Pred, OpParent)) {
- AddCosts(Op);
- continue;
- }
+ if (uint64_t Scale = CheckPredecessors(Parent, Pred, OpParent))
+ AddCosts(Op, Scale);
}
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-loop-backedge.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-loop-backedge.ll
index 29e6a3a354354..9b1c9e977d6a1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-loop-backedge.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-loop-backedge.ll
@@ -1,26 +1,18 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -pass-remarks-output=%t < %s | FileCheck %s
-; RUN: FileCheck --input-file=%t %s --check-prefix=YAML
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
declare void @external_call()
-; YAML: --- !Passed
-; YAML-NEXT: Pass: slp-vectorizer
-; YAML-NEXT: Name: StoresVectorized
-; YAML-NEXT: Function: test_spillcost_backedge
-; YAML-NEXT: Args:
-; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-99'
-; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '6'
-; YAML-NEXT: ...
+; The vectorized fdiv pair is loop-invariant and emitted in %entry, so its
+; vector value is live across the call in the loop body and must be
+; spilled/reloaded around it on every iteration. The spill cost scaled by
+; the loop trip count makes this tree unprofitable, so it must stay scalar.
define void @test_spillcost_backedge(ptr noalias %res, ptr noalias %in, double %x, double %y) {
; CHECK-LABEL: define void @test_spillcost_backedge(
; CHECK-SAME: ptr noalias [[RES:%.*]], ptr noalias [[IN:%.*]], double [[X:%.*]], double [[Y:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[X]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[Y]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = fdiv <2 x double> splat (double 1.000000e+00), [[TMP1]]
+; CHECK-NEXT: [[A:%.*]] = fdiv double 1.000000e+00, [[X]]
+; CHECK-NEXT: [[B:%.*]] = fdiv double 1.000000e+00, [[Y]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
@@ -28,10 +20,15 @@ define void @test_spillcost_backedge(ptr noalias %res, ptr noalias %in, double %
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_BODY:.*]], label %[[EXIT:.*]]
; CHECK: [[LOOP_BODY]]:
; CHECK-NEXT: [[GEP_IN_0:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_IN_1:%.*]] = getelementptr inbounds double, ptr [[GEP_IN_0]], i64 1
+; CHECK-NEXT: [[V0:%.*]] = load double, ptr [[GEP_IN_0]], align 8
+; CHECK-NEXT: [[V1:%.*]] = load double, ptr [[GEP_IN_1]], align 8
+; CHECK-NEXT: [[R1:%.*]] = fsub double [[V0]], [[A]]
+; CHECK-NEXT: [[R2:%.*]] = fsub double [[V1]], [[B]]
; CHECK-NEXT: [[GEP_OUT_0:%.*]] = getelementptr double, ptr [[RES]], i64 [[IV]]
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[GEP_IN_0]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], [[TMP2]]
-; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[GEP_OUT_0]], align 8
+; CHECK-NEXT: [[GEP_OUT_1:%.*]] = getelementptr inbounds double, ptr [[GEP_OUT_0]], i64 1
+; CHECK-NEXT: store double [[R1]], ptr [[GEP_OUT_0]], align 8
+; CHECK-NEXT: store double [[R2]], ptr [[GEP_OUT_1]], align 8
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[IV]], 50
; CHECK-NEXT: br i1 [[COND]], label %[[CALL_BLOCK:.*]], label %[[LOOP_LATCH]]
; CHECK: [[CALL_BLOCK]]:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/spillcost-loop-invariant-entry.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/spillcost-loop-invariant-entry.ll
index 01b69b46af8ea..42ecb4a3d7898 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/spillcost-loop-invariant-entry.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/spillcost-loop-invariant-entry.ll
@@ -13,26 +13,28 @@ define void @invariant_entry_over_call(ptr %out, ptr %in, ptr %sink, double %a,
; CHECK-LABEL: define void @invariant_entry_over_call(
; CHECK-SAME: ptr [[OUT:%.*]], ptr [[IN:%.*]], ptr [[SINK:%.*]], double [[A:%.*]], double [[B:%.*]], double [[C:%.*]], double [[D:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[B]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[C]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[D]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP4]])
+; CHECK-NEXT: [[DIV0:%.*]] = fdiv double [[A]], [[C]]
+; CHECK-NEXT: [[DIV1:%.*]] = fdiv double [[B]], [[D]]
+; CHECK-NEXT: [[SQ0:%.*]] = call double @llvm.sqrt.f64(double [[DIV0]])
+; CHECK-NEXT: [[SQ1:%.*]] = call double @llvm.sqrt.f64(double [[DIV1]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds double, ptr [[IN]], i64 [[I]]
+; CHECK-NEXT: [[V0:%.*]] = load double, ptr [[P0]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[V1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul double [[V0]], [[SQ0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul double [[V1]], [[SQ1]]
; CHECK-NEXT: [[O0:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[I]]
-; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr [[P0]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x double> [[TMP6]], [[TMP5]]
-; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[O0]], align 8
+; CHECK-NEXT: store double [[M0]], ptr [[O0]], align 8
+; CHECK-NEXT: [[O1:%.*]] = getelementptr inbounds double, ptr [[O0]], i64 1
+; CHECK-NEXT: store double [[M1]], ptr [[O1]], align 8
; CHECK-NEXT: call void @g()
; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I]], 2
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INC]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP7]], i64 1
; CHECK-NEXT: store double [[M1]], ptr [[SINK]], align 8
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list