[llvm] [SLP] De-duplicate costing of perfect-diamond gather spills (PR #220569)
Hari Limaye via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 04:09:44 PDT 2026
https://github.com/hazzlim updated https://github.com/llvm/llvm-project/pull/220569
>From 796c7b69f161cdbc6fe799736ed0c2a226a60e13 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 2 Sep 2026 10:11:56 +0000
Subject: [PATCH 01/10] [SLP] De-duplicate costing of perfect-diamond gather
spills
Refactor the logic in `getSpillCost` to avoid double counting the cost
of spilling a loop-invariant perfect-diamond gather entry across a loop
containing a call.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 70 +++++++++++++++----
.../spillcost-shared-loop-invariant.ll | 44 ++++--------
2 files changed, 72 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3d1c305660102..30c31b84c1ff8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18936,26 +18936,33 @@ InstructionCost BoUpSLP::getSpillCost() {
LoopBodyHasNonVecCall.try_emplace(L, false);
return false;
};
- auto CheckPredecessors = [&](BasicBlock *Root, BasicBlock *Pred,
- BasicBlock *OpParent) {
- auto Key = std::make_pair(Root, OpParent);
- if (auto It = ParentOpParentToPreds.find(Key);
- It != ParentOpParentToPreds.end())
- return It->second;
- uint64_t Res = 0;
- scope_exit Cleanup([&]() { ParentOpParentToPreds.try_emplace(Key, Res); });
+ auto GetLoopInvariantSpillRegion = [&](BasicBlock *UseBB,
+ BasicBlock *DefBB) -> const Loop * {
// If Op is loop-invariant, a call anywhere in the loop body forces a spill,
// even when a call-free forward path from Root back to OpParent exists on
// the first iteration. Find the outermost such enclosing loop and reject if
// its body contains a non-vec call.
- const Loop *L = LI->getLoopFor(Root);
+ const Loop *L = LI->getLoopFor(UseBB);
const Loop *Outermost = nullptr;
- while (L && !L->contains(OpParent)) {
+ while (L && !L->contains(DefBB)) {
Outermost = L;
L = L->getParentLoop();
}
if (Outermost && LoopBodyHasCall(Outermost)) {
- Res = getLoopNestScale(Outermost);
+ return Outermost;
+ }
+ return nullptr;
+ };
+ auto CheckPredecessors = [&](BasicBlock *Root, BasicBlock *Pred,
+ BasicBlock *OpParent) {
+ auto Key = std::make_pair(Root, OpParent);
+ if (auto It = ParentOpParentToPreds.find(Key);
+ It != ParentOpParentToPreds.end())
+ return It->second;
+ uint64_t Res = 0;
+ scope_exit Cleanup([&]() { ParentOpParentToPreds.try_emplace(Key, Res); });
+ if (const auto *L = GetLoopInvariantSpillRegion(Root, OpParent)) {
+ Res = getLoopNestScale(L);
return Res;
}
SmallVector<BasicBlock *> Worklist;
@@ -19040,6 +19047,44 @@ InstructionCost BoUpSLP::getSpillCost() {
}
return nullptr;
};
+ auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
+ const Loop *SpillLoop) -> bool {
+ assert(Gather->isGather() && Gather->hasState());
+
+ // Find the real vector entry reused by this perfect-diamond gather.
+ const TreeEntry *SameTE = getSameValuesTreeEntry(
+ Gather->getMainOp(), Gather->Scalars, /*SameVF=*/true);
+ if (!SameTE || SameTE == Gather || SameTE->State != TreeEntry::Vectorize ||
+ ScalarOrPseudoEntries.contains(SameTE) || !SameTE->UserTreeIndex)
+ return false;
+
+ // Only permit an ordinary vectorized user.
+ const TreeEntry *UserTE = SameTE->UserTreeIndex.UserTE;
+ if (!UserTE || UserTE->State != TreeEntry::Vectorize ||
+ ScalarOrPseudoEntries.contains(UserTE) ||
+ UserTE->getOpcode() == Instruction::PHI)
+ return false;
+
+ Instruction *Def = EntriesToLastInstruction.lookup(SameTE);
+ Instruction *Use = EntriesToLastInstruction.lookup(UserTE);
+ if (!Def || !Use)
+ return false;
+
+ // Require the matching entry to be defined outside the loop and its use to
+ // execute directly in the same loop as the gather user.
+ if (SpillLoop->contains(Def->getParent()) ||
+ LI->getLoopFor(Use->getParent()) != SpillLoop)
+ return false;
+
+ // Ensure that the matching entry is also loop invariant.
+ if (!all_of(SameTE->Scalars, [&](Value *V) {
+ return !isa<Instruction>(V) || SpillLoop->isLoopInvariant(V);
+ }))
+ return false;
+
+ return GetLoopInvariantSpillRegion(Use->getParent(), Def->getParent()) ==
+ SpillLoop;
+ };
while (!LiveEntries.empty()) {
const TreeEntry *Entry = LiveEntries.pop_back_val();
const auto OpIt = EntriesToOperands.find(Entry);
@@ -19079,7 +19124,8 @@ InstructionCost BoUpSLP::getSpillCost() {
all_of(Op->Scalars, [&](Value *V) {
return !isa<Instruction>(V) || L->isLoopInvariant(V);
}))
- AddCosts(Op, GetSpillScale(Parent));
+ if (!IsCoveredByMatchingVectorEntry(Op, L))
+ AddCosts(Op, GetSpillScale(Parent));
continue;
}
Budget = 0;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-shared-loop-invariant.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-shared-loop-invariant.ll
index 0ce0fba110477..706b6afd565d6 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-shared-loop-invariant.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-shared-loop-invariant.ll
@@ -9,27 +9,20 @@ define void @shared_loop_invariant(ptr %params, i1 %cond, i64 %n) {
; CHECK-SAME: ptr [[PARAMS:%.*]], i1 [[COND:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[OFFSET:%.*]] = getelementptr i8, ptr [[PARAMS]], i64 88
-; CHECK-NEXT: [[X:%.*]] = load double, ptr [[OFFSET]], align 8
-; CHECK-NEXT: [[Y_PTR:%.*]] = getelementptr i8, ptr [[PARAMS]], i64 96
-; CHECK-NEXT: [[Y:%.*]] = load double, ptr [[Y_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[OFFSET]], align 8
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
; CHECK-NEXT: br i1 [[COND]], label %[[RIGHT:.*]], label %[[LEFT:.*]]
; CHECK: [[LEFT]]:
-; CHECK-NEXT: [[LEFT_X:%.*]] = fadd double 0.000000e+00, [[X]]
-; CHECK-NEXT: [[LEFT_Y:%.*]] = fadd double 0.000000e+00, [[Y]]
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> zeroinitializer, [[TMP0]]
; CHECK-NEXT: br label %[[MERGE]]
; CHECK: [[RIGHT]]:
-; CHECK-NEXT: [[RIGHT_X:%.*]] = fadd double 0.000000e+00, [[X]]
-; CHECK-NEXT: [[RIGHT_Y:%.*]] = fadd double 0.000000e+00, [[Y]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> zeroinitializer, [[TMP0]]
; CHECK-NEXT: br label %[[MERGE]]
; CHECK: [[MERGE]]:
-; CHECK-NEXT: [[PHI_Y:%.*]] = phi double [ [[RIGHT_Y]], %[[RIGHT]] ], [ [[LEFT_Y]], %[[LEFT]] ]
-; CHECK-NEXT: [[PHI_X:%.*]] = phi double [ [[RIGHT_X]], %[[RIGHT]] ], [ [[LEFT_X]], %[[LEFT]] ]
-; CHECK-NEXT: store double [[PHI_X]], ptr [[PARAMS]], align 8
-; CHECK-NEXT: [[OUT_Y:%.*]] = getelementptr i8, ptr [[PARAMS]], i64 8
-; CHECK-NEXT: store double [[PHI_Y]], ptr [[OUT_Y]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x double> [ [[TMP2]], %[[RIGHT]] ], [ [[TMP1]], %[[LEFT]] ]
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[PARAMS]], align 8
; CHECK-NEXT: call void @external()
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i64 [[NEXT]], [[N]]
@@ -78,33 +71,24 @@ define void @shared_coordinate_offsets(ptr %params, ptr %left.coords, ptr %right
; CHECK-SAME: ptr [[PARAMS:%.*]], ptr [[LEFT_COORDS:%.*]], ptr [[RIGHT_COORDS:%.*]], ptr [[OUT:%.*]], i1 [[COND:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[OFFSET_PTR:%.*]] = getelementptr i8, ptr [[PARAMS]], i64 88
-; CHECK-NEXT: [[OFFSET_X:%.*]] = load double, ptr [[OFFSET_PTR]], align 8
-; CHECK-NEXT: [[OFFSET_Y_PTR:%.*]] = getelementptr i8, ptr [[PARAMS]], i64 96
-; CHECK-NEXT: [[OFFSET_Y:%.*]] = load double, ptr [[OFFSET_Y_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[OFFSET_PTR]], align 8
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
; CHECK-NEXT: br i1 [[COND]], label %[[RIGHT:.*]], label %[[LEFT:.*]]
; CHECK: [[LEFT]]:
-; CHECK-NEXT: [[LEFT_X:%.*]] = load double, ptr [[LEFT_COORDS]], align 8
-; CHECK-NEXT: [[LEFT_Y_PTR:%.*]] = getelementptr i8, ptr [[LEFT_COORDS]], i64 8
-; CHECK-NEXT: [[LEFT_Y:%.*]] = load double, ptr [[LEFT_Y_PTR]], align 8
-; CHECK-NEXT: [[LEFT_ADJUSTED_X:%.*]] = fadd fast double [[LEFT_X]], [[OFFSET_X]]
-; CHECK-NEXT: [[LEFT_ADJUSTED_Y:%.*]] = fadd fast double [[LEFT_Y]], [[OFFSET_Y]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[LEFT_COORDS]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP0]]
; CHECK-NEXT: br label %[[MERGE]]
; CHECK: [[RIGHT]]:
-; CHECK-NEXT: [[RIGHT_X:%.*]] = load double, ptr [[RIGHT_COORDS]], align 8
-; CHECK-NEXT: [[RIGHT_Y_PTR:%.*]] = getelementptr i8, ptr [[RIGHT_COORDS]], i64 8
-; CHECK-NEXT: [[RIGHT_Y:%.*]] = load double, ptr [[RIGHT_Y_PTR]], align 8
-; CHECK-NEXT: [[RIGHT_ADJUSTED_X:%.*]] = fadd fast double [[RIGHT_X]], [[OFFSET_X]]
-; CHECK-NEXT: [[RIGHT_ADJUSTED_Y:%.*]] = fadd fast double [[RIGHT_Y]], [[OFFSET_Y]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[RIGHT_COORDS]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP0]]
; CHECK-NEXT: br label %[[MERGE]]
; CHECK: [[MERGE]]:
-; CHECK-NEXT: [[ADJUSTED_X:%.*]] = phi double [ [[RIGHT_ADJUSTED_X]], %[[RIGHT]] ], [ [[LEFT_ADJUSTED_X]], %[[LEFT]] ]
-; CHECK-NEXT: [[ADJUSTED_Y:%.*]] = phi double [ [[RIGHT_ADJUSTED_Y]], %[[RIGHT]] ], [ [[LEFT_ADJUSTED_Y]], %[[LEFT]] ]
-; CHECK-NEXT: store double [[ADJUSTED_X]], ptr [[OUT]], align 8
-; CHECK-NEXT: [[OUT_Y:%.*]] = getelementptr i8, ptr [[OUT]], i64 8
-; CHECK-NEXT: store double [[ADJUSTED_Y]], ptr [[OUT_Y]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x double> [ [[TMP4]], %[[RIGHT]] ], [ [[TMP2]], %[[LEFT]] ]
+; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[ADJUSTED_X:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT: [[ADJUSTED_Y:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
; CHECK-NEXT: call void @use_coordinates(double [[ADJUSTED_X]], double [[ADJUSTED_Y]])
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ult i64 [[NEXT]], [[N]]
>From a287182e84f9010520b08731388f961f5aef48ba Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 2 Sep 2026 14:14:59 +0000
Subject: [PATCH 02/10] Fixup
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 30c31b84c1ff8..e92e17a61b825 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19049,7 +19049,9 @@ InstructionCost BoUpSLP::getSpillCost() {
};
auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
const Loop *SpillLoop) -> bool {
- assert(Gather->isGather() && Gather->hasState());
+ assert(Gather->isGather());
+ if (!Gather->hasState())
+ return false;
// Find the real vector entry reused by this perfect-diamond gather.
const TreeEntry *SameTE = getSameValuesTreeEntry(
>From 893edd6a018e0cf2457d1f5e375c09695aedc012 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 2 Sep 2026 21:46:45 +0000
Subject: [PATCH 03/10] Review suggested changes
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 ++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e92e17a61b825..66c6ccaba77f9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18938,10 +18938,11 @@ InstructionCost BoUpSLP::getSpillCost() {
};
auto GetLoopInvariantSpillRegion = [&](BasicBlock *UseBB,
BasicBlock *DefBB) -> const Loop * {
- // If Op is loop-invariant, a call anywhere in the loop body forces a spill,
- // even when a call-free forward path from Root back to OpParent exists on
- // the first iteration. Find the outermost such enclosing loop and reject if
- // its body contains a non-vec call.
+ // If DefBB is outside a loop containing UseBB, the value is live across
+ // iterations of that loop. A non-vectorized call in the loop may therefore
+ // require a spill even when the entry path from DefBB to UseBB is
+ // call-free. Return the outermost such loop containing a relevant call so
+ // its execution scale can be used to cost the spill.
const Loop *L = LI->getLoopFor(UseBB);
const Loop *Outermost = nullptr;
while (L && !L->contains(DefBB)) {
@@ -19067,6 +19068,17 @@ InstructionCost BoUpSLP::getSpillCost() {
UserTE->getOpcode() == Instruction::PHI)
return false;
+ // The spill walk does not descend through gather entries; if any ancestor
+ // of the matching entry's user is a gather, the matching edge is never
+ // charged and de-duplicating would lose the spill cost entirely.
+ for (const TreeEntry *E = UserTE; E != Root;) {
+ if (!E->UserTreeIndex)
+ return false;
+ E = E->UserTreeIndex.UserTE;
+ if (E->isGather())
+ return false;
+ }
+
Instruction *Def = EntriesToLastInstruction.lookup(SameTE);
Instruction *Use = EntriesToLastInstruction.lookup(UserTE);
if (!Def || !Use)
@@ -19084,6 +19096,12 @@ InstructionCost BoUpSLP::getSpillCost() {
}))
return false;
+ // A non-vec call between Def and the end of its block preempts the
+ // loop-invariant charge for the matching entry's edge with a smaller
+ // scale, so de-duplicating would drop the in-loop spill cost.
+ if (!CheckForNonVecCallsInSameBlock(Def, Def->getParent()->getTerminator()))
+ return false;
+
return GetLoopInvariantSpillRegion(Use->getParent(), Def->getParent()) ==
SpillLoop;
};
>From eafa8116eb4d16b1e944090a612350b4bcb34350 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 3 Sep 2026 10:46:49 +0000
Subject: [PATCH 04/10] Don't require Gather to have state
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 66c6ccaba77f9..6b338d4699b78 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19051,12 +19051,10 @@ InstructionCost BoUpSLP::getSpillCost() {
auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
const Loop *SpillLoop) -> bool {
assert(Gather->isGather());
- if (!Gather->hasState())
- return false;
// Find the real vector entry reused by this perfect-diamond gather.
const TreeEntry *SameTE = getSameValuesTreeEntry(
- Gather->getMainOp(), Gather->Scalars, /*SameVF=*/true);
+ Gather->Scalars.front(), Gather->Scalars, /*SameVF=*/true);
if (!SameTE || SameTE == Gather || SameTE->State != TreeEntry::Vectorize ||
ScalarOrPseudoEntries.contains(SameTE) || !SameTE->UserTreeIndex)
return false;
>From 1c234d6dd758f48e6ad3300256b68337fd6caab1 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 3 Sep 2026 15:51:43 +0000
Subject: [PATCH 05/10] Use MainOp or first non-poison/non-undef
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6b338d4699b78..05dd4e2a6c96a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19052,9 +19052,21 @@ InstructionCost BoUpSLP::getSpillCost() {
const Loop *SpillLoop) -> bool {
assert(Gather->isGather());
+ Value *LookupValue = nullptr;
+ if (Gather->hasState()) {
+ LookupValue = Gather->getMainOp();
+ } else {
+ auto *It = find_if(Gather->Scalars, [](Value *V) {
+ return !isa<PoisonValue, UndefValue>(V);
+ });
+ if (It == Gather->Scalars.end())
+ return false;
+ LookupValue = *It;
+ }
+
// Find the real vector entry reused by this perfect-diamond gather.
- const TreeEntry *SameTE = getSameValuesTreeEntry(
- Gather->Scalars.front(), Gather->Scalars, /*SameVF=*/true);
+ const TreeEntry *SameTE =
+ getSameValuesTreeEntry(LookupValue, Gather->Scalars, /*SameVF=*/true);
if (!SameTE || SameTE == Gather || SameTE->State != TreeEntry::Vectorize ||
ScalarOrPseudoEntries.contains(SameTE) || !SameTE->UserTreeIndex)
return false;
>From 6fc9ab4d81332c409910ce2cc6decea997cdb58e Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Fri, 4 Sep 2026 09:35:18 +0000
Subject: [PATCH 06/10] Review updates
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 05dd4e2a6c96a..37393f11a6f07 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18949,10 +18949,7 @@ InstructionCost BoUpSLP::getSpillCost() {
Outermost = L;
L = L->getParentLoop();
}
- if (Outermost && LoopBodyHasCall(Outermost)) {
- return Outermost;
- }
- return nullptr;
+ return (Outermost && LoopBodyHasCall(Outermost)) ? Outermost : nullptr;
};
auto CheckPredecessors = [&](BasicBlock *Root, BasicBlock *Pred,
BasicBlock *OpParent) {
@@ -19073,11 +19070,16 @@ InstructionCost BoUpSLP::getSpillCost() {
// Only permit an ordinary vectorized user.
const TreeEntry *UserTE = SameTE->UserTreeIndex.UserTE;
- if (!UserTE || UserTE->State != TreeEntry::Vectorize ||
+ assert(UserTE && "Expected a user tree entry.");
+ if (UserTE->State != TreeEntry::Vectorize ||
ScalarOrPseudoEntries.contains(UserTE) ||
UserTE->getOpcode() == Instruction::PHI)
return false;
+ // Different demotion state would make the two edge costs unequal.
+ if (MinBWs.contains(SameTE) != MinBWs.contains(Gather))
+ return false;
+
// The spill walk does not descend through gather entries; if any ancestor
// of the matching entry's user is a gather, the matching edge is never
// charged and de-duplicating would lose the spill cost entirely.
>From 14b5737a8e1a277f8968979b3e3c23aa4caa8eac Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Mon, 7 Sep 2026 08:42:30 +0000
Subject: [PATCH 07/10] Move the budget reset before we do any analysis
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 37393f11a6f07..48cd8443fa9c8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19145,6 +19145,10 @@ InstructionCost BoUpSLP::getSpillCost() {
if (Entry->State == TreeEntry::SplitVectorize ||
(Op->isGather() && allConstant(Op->Scalars)))
continue;
+
+ // Reset the scan budget for analsysis of each edge.
+ Budget = 0;
+
// A gather with all loop-invariant lanes is hoisted to the loop
// preheader by optimizeGatherSequence, so its vector value becomes live
// across any non-vectorized call in the loop body. Charge it like any
@@ -19160,7 +19164,6 @@ InstructionCost BoUpSLP::getSpillCost() {
AddCosts(Op, GetSpillScale(Parent));
continue;
}
- Budget = 0;
BasicBlock *Pred = nullptr;
if (auto *Phi = dyn_cast<PHINode>(Entry->getMainOp()))
Pred = Phi->getIncomingBlock(Op->UserTreeIndex.EdgeIdx);
>From be0a5f6fac6284ce5e34c6abf004ec5bd9a1e316 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 8 Sep 2026 10:29:19 +0000
Subject: [PATCH 08/10] Review updates
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 48cd8443fa9c8..69873bbb14267 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19047,15 +19047,13 @@ InstructionCost BoUpSLP::getSpillCost() {
};
auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
const Loop *SpillLoop) -> bool {
- assert(Gather->isGather());
+ assert(Gather->isGather() && "Expected a Gather Entry!");
Value *LookupValue = nullptr;
if (Gather->hasState()) {
LookupValue = Gather->getMainOp();
} else {
- auto *It = find_if(Gather->Scalars, [](Value *V) {
- return !isa<PoisonValue, UndefValue>(V);
- });
+ auto *It = find_if_not(Gather->Scalars, IsaPred<UndefValue>);
if (It == Gather->Scalars.end())
return false;
LookupValue = *It;
@@ -19081,13 +19079,14 @@ InstructionCost BoUpSLP::getSpillCost() {
return false;
// The spill walk does not descend through gather entries; if any ancestor
- // of the matching entry's user is a gather, the matching edge is never
- // charged and de-duplicating would lose the spill cost entirely.
+ // of the matching entry's user is a gather or a combined scalar/pseudo
+ // entry, the matching edge is never charged and de-duplicating would lose
+ // the spill cost entirely.
for (const TreeEntry *E = UserTE; E != Root;) {
if (!E->UserTreeIndex)
return false;
E = E->UserTreeIndex.UserTE;
- if (E->isGather())
+ if (E->isGather() || ScalarOrPseudoEntries.contains(SameTE))
return false;
}
>From 862c373a9807a302244f172044ec8389415fbc47 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 8 Sep 2026 15:08:55 +0000
Subject: [PATCH 09/10] Address Review
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 69873bbb14267..9b5daed501859 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19047,7 +19047,7 @@ InstructionCost BoUpSLP::getSpillCost() {
};
auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
const Loop *SpillLoop) -> bool {
- assert(Gather->isGather() && "Expected a Gather Entry!");
+ assert(Gather->isGather() && "Expected a gather/buildvector entry.");
Value *LookupValue = nullptr;
if (Gather->hasState()) {
@@ -19075,7 +19075,7 @@ InstructionCost BoUpSLP::getSpillCost() {
return false;
// Different demotion state would make the two edge costs unequal.
- if (MinBWs.contains(SameTE) != MinBWs.contains(Gather))
+ if (MinBWs.lookup(SameTE) != MinBWs.lookup(Gather))
return false;
// The spill walk does not descend through gather entries; if any ancestor
@@ -19086,14 +19086,12 @@ InstructionCost BoUpSLP::getSpillCost() {
if (!E->UserTreeIndex)
return false;
E = E->UserTreeIndex.UserTE;
- if (E->isGather() || ScalarOrPseudoEntries.contains(SameTE))
+ if (E->isGather() || ScalarOrPseudoEntries.contains(E))
return false;
}
- Instruction *Def = EntriesToLastInstruction.lookup(SameTE);
- Instruction *Use = EntriesToLastInstruction.lookup(UserTE);
- if (!Def || !Use)
- return false;
+ Instruction *Def = EntriesToLastInstruction.at(SameTE);
+ Instruction *Use = EntriesToLastInstruction.at(UserTE);
// Require the matching entry to be defined outside the loop and its use to
// execute directly in the same loop as the gather user.
@@ -19145,7 +19143,7 @@ InstructionCost BoUpSLP::getSpillCost() {
(Op->isGather() && allConstant(Op->Scalars)))
continue;
- // Reset the scan budget for analsysis of each edge.
+ // Reset the scan budget for analysis of each edge.
Budget = 0;
// A gather with all loop-invariant lanes is hoisted to the loop
>From 39bf19262228b5bea5e2b9548e00c79d0c022091 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 9 Sep 2026 11:03:57 +0000
Subject: [PATCH 10/10] Final review
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9b5daed501859..4a6fc05634109 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19066,6 +19066,12 @@ InstructionCost BoUpSLP::getSpillCost() {
ScalarOrPseudoEntries.contains(SameTE) || !SameTE->UserTreeIndex)
return false;
+ // A reordered or reuse-shuffled match materializes a separate shuffled
+ // vector that is itself live across the call; only an exact match reuses
+ // the same vector value.
+ if (!SameTE->ReorderIndices.empty() || !SameTE->ReuseShuffleIndices.empty())
+ return false;
+
// Only permit an ordinary vectorized user.
const TreeEntry *UserTE = SameTE->UserTreeIndex.UserTE;
assert(UserTE && "Expected a user tree entry.");
@@ -19074,8 +19080,8 @@ InstructionCost BoUpSLP::getSpillCost() {
UserTE->getOpcode() == Instruction::PHI)
return false;
- // Different demotion state would make the two edge costs unequal.
- if (MinBWs.lookup(SameTE) != MinBWs.lookup(Gather))
+ // Different demoted bitwidth would make the two edge costs unequal.
+ if (MinBWs.lookup(SameTE).first != MinBWs.lookup(Gather).first)
return false;
// The spill walk does not descend through gather entries; if any ancestor
More information about the llvm-commits
mailing list