[llvm] b2f9210 - [SLP] Keep loops BTCs across CurrentLoopNest truncations
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 21:40:28 PDT 2026
Author: Alexey Bataev
Date: 2026-05-02T00:40:23-04:00
New Revision: b2f9210e0bc4e76d9fae807edd81421d9df0e469
URL: https://github.com/llvm/llvm-project/commit/b2f9210e0bc4e76d9fae807edd81421d9df0e469
DIFF: https://github.com/llvm/llvm-project/commit/b2f9210e0bc4e76d9fae807edd81421d9df0e469.diff
LOG: [SLP] Keep loops BTCs across CurrentLoopNest truncations
Record SCEV BTCs in a per-depth vector so a later loop nest reaching a
previously merged depth via the empty, divergence, or extend branch in
buildTreeRec is re-validated.
Reviewers:
Pull Request: https://github.com/llvm/llvm-project/pull/195411
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/sibling-loops-mismatched-tripcount.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3030c12463a54..d3b6583f34a96 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2287,6 +2287,7 @@ class slpvectorizer::BoUpSLP {
ValueToGatherNodes.clear();
TreeEntryToStridedPtrInfoMap.clear();
CurrentLoopNest.clear();
+ MergedLoopBTCs.clear();
}
unsigned getTreeSize() const { return VectorizableTree.size(); }
@@ -4930,6 +4931,10 @@ class slpvectorizer::BoUpSLP {
/// multiple, to avoid side-effects from the loop-aware cost model.
SmallVector<const Loop *> CurrentLoopNest;
+ /// Per-depth SCEVs trip counts at every loop level where the tree builder has
+ /// joined diverging sibling loops.
+ SmallVector<const SCEV *> MergedLoopBTCs;
+
/// Maps the loops to their loop nests.
SmallDenseMap<const Loop *, SmallVector<const Loop *>> LoopToLoopNest;
@@ -12694,7 +12699,29 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
break;
++CommonLen;
}
+ auto ValidateMergedBTCs = [&](unsigned StartDepth) -> bool {
+ unsigned EndDepth =
+ std::min<unsigned>(NewLoopNest.size(), MergedLoopBTCs.size());
+ for (unsigned D = StartDepth; D < EndDepth; ++D) {
+ const SCEV *Constraint = MergedLoopBTCs[D];
+ if (!Constraint)
+ continue;
+ const SCEV *NewBTC = SE->getBackedgeTakenCount(NewLoopNest[D]);
+ if (isa<SCEVCouldNotCompute>(NewBTC) || NewBTC != Constraint)
+ return false;
+ }
+ return true;
+ };
+ auto BailOutToGather = [&]() {
+ LLVM_DEBUG(dbgs()
+ << "SLP: Sibling loops have
diff erent trip counts.\n");
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
+ };
if (CurrentLoopNest.empty()) {
+ if (!ValidateMergedBTCs(0)) {
+ BailOutToGather();
+ return;
+ }
CurrentLoopNest.assign(NewLoopNest);
} else if (CommonLen < CurrentLoopNest.size() &&
CommonLen < NewLoopNest.size()) {
@@ -12711,14 +12738,22 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
const SCEV *BecA = SE->getBackedgeTakenCount(SibA);
const SCEV *BecB = SE->getBackedgeTakenCount(SibB);
if (isa<SCEVCouldNotCompute>(BecA) || BecA != BecB) {
- LLVM_DEBUG(dbgs()
- << "SLP: Sibling loops have
diff erent trip counts.\n");
- newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
+ BailOutToGather();
return;
}
+ if (!ValidateMergedBTCs(CommonLen + 1)) {
+ BailOutToGather();
+ return;
+ }
+ if (MergedLoopBTCs.size() <= CommonLen)
+ MergedLoopBTCs.resize(CommonLen + 1, nullptr);
+ MergedLoopBTCs[CommonLen] = BecA;
CurrentLoopNest.truncate(CommonLen);
} else if (NewLoopNest.size() > CurrentLoopNest.size()) {
- // New entry lives deeper in the same nest chain; extend.
+ if (!ValidateMergedBTCs(CurrentLoopNest.size())) {
+ BailOutToGather();
+ return;
+ }
CurrentLoopNest.append(
std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
NewLoopNest.end());
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sibling-loops-mismatched-tripcount.ll b/llvm/test/Transforms/SLPVectorizer/X86/sibling-loops-mismatched-tripcount.ll
index 68b067c2d035c..6f138b32f845f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sibling-loops-mismatched-tripcount.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sibling-loops-mismatched-tripcount.ll
@@ -9,18 +9,36 @@
; YAML-NEXT: Function: sibling_root_loops_mismatched_tripcount
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '31'
+; YAML-NEXT: - Cost: '27'
; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '11'
+; YAML-NEXT: - TreeSize: '9'
; YAML: --- !Passed
; YAML-NEXT: Pass: slp-vectorizer
; YAML-NEXT: Name: StoresVectorized
; YAML-NEXT: Function: sibling_root_loops_mismatched_tripcount
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '29'
+; YAML-NEXT: - Cost: '25'
; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '11'
+; YAML-NEXT: - TreeSize: '9'
+; YAML: --- !Passed
+; YAML-NEXT: Pass: slp-vectorizer
+; YAML-NEXT: Name: VectorizedList
+; YAML-NEXT: Function: sibling_root_loops_mismatched_tripcount
+; YAML-NEXT: Args:
+; YAML-NEXT: - String: 'SLP vectorized with cost '
+; YAML-NEXT: - Cost: '4'
+; YAML-NEXT: - String: ' and with tree size '
+; YAML-NEXT: - TreeSize: '4'
+; YAML: --- !Passed
+; YAML-NEXT: Pass: slp-vectorizer
+; YAML-NEXT: Name: VectorizedList
+; YAML-NEXT: Function: sibling_root_loops_mismatched_tripcount
+; YAML-NEXT: Args:
+; YAML-NEXT: - String: 'SLP vectorized with cost '
+; YAML-NEXT: - Cost: '4'
+; YAML-NEXT: - String: ' and with tree size '
+; YAML-NEXT: - TreeSize: '4'
define void @sibling_root_loops_mismatched_tripcount(ptr %dst, i64 %n, i64 %m, i32 %sel) {
; CHECK-LABEL: define void @sibling_root_loops_mismatched_tripcount(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i32 [[SEL:%.*]]) {
@@ -125,7 +143,7 @@ merge:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
; YAML-NEXT: - Cost: '200'
; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '11'
+; YAML-NEXT: - TreeSize: '9'
; YAML: --- !Passed
; YAML-NEXT: Pass: slp-vectorizer
; YAML-NEXT: Name: StoresVectorized
@@ -134,7 +152,7 @@ merge:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
; YAML-NEXT: - Cost: '200'
; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '11'
+; YAML-NEXT: - TreeSize: '9'
define void @sibling_inner_loops_mismatched_tripcount(ptr %dst, i64 %n, i64 %m, i32 %sel) {
; CHECK-LABEL: define void @sibling_inner_loops_mismatched_tripcount(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i32 [[SEL:%.*]]) {
More information about the llvm-commits
mailing list